2.1_nBayes_remove_transfusion_terms.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.1_nBayes_remove_transfusion_terms\n",
    "## Clean up top 5k  terms \n",
    "- needs to be run on sparse matrix of 5000 terms from naive bayes model output from 2.0_classification_models.ipynb \n",
    "- used where the **model.feature_log_prob_** was used to rank terms (not coefs). \n",
    "run with python 2.7\n",
    "\n",
    "\n",
    "    1. load, extract \n",
    "    2. remove transfusion only terms\n",
    "    3. collapse to longest n-gram because it merges them and creates terms that are not in our original list ... which means we can't 'drop them from the model b/c they don't exist\n",
    "    4. save"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0 import packages and initialize database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python 2 & 3 Compatibility\n",
    "from __future__ import print_function, division"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "from fuzzywuzzy import fuzz, process\n",
    "from difflib import SequenceMatcher\n",
    "import itertools\n",
    "import scipy\n",
    "import psycopg2\n",
    "from sqlalchemy import create_engine\n",
    "import sys\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "\n",
    "sys.setrecursionlimit(10000)\n",
    "#pd.set_option('display.max_rows', 1000)\n",
    "#pd.set_option('display.max_colwidth', -1)\n",
    "\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "conn = psycopg2.connect(\"dbname=mimic user=xxxxxxxxxxx options=--search_path=mimiciii\");\n",
    "engine = create_engine('postgresql://xxxxxxxxxxx@localhost/mimic')\n",
    "path1 = './'\n",
    "    \n",
    "\n",
    "cur = conn.cursor();\n",
    "cur.execute(\"\"\"SET search_path = mimiciii;\"\"\")\n",
    "\n",
    "from importlib_metadata import version\n",
    "\n",
    "\n",
    "libraries = ['pandas','sqlalchemy','psycopg2','tqdm','numpy', 'scipy','fuzzywuzzy','seaborn','matplotlib']\n",
    "print('last ran: ',datetime.now() )\n",
    "print(\"Python Version:\", sys.version[0:7])\n",
    "print( \"operating system:\", sys.platform)\n",
    "\n",
    "for lib in libraries:\n",
    "    print(lib + ' version: ' + version(lib))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.0 Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('NB_top_5000_matrix.pickle', 'rb') as f:\n",
    "\n",
    "    mat,terms0=pickle.load(f)\n",
    "\n",
    "terms=list(terms0.iloc[:,0])\n",
    "\n",
    "print(mat.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 get coo \n",
    "- Return a Coordinate (coo) representation of the Compresses-Sparse-Column (csc) matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coo = mat.tocoo(copy=False)\n",
    "\n",
    "# Access `row`, `col` and `data` properties of coo matrix.\n",
    "mat = pd.DataFrame({'feature': coo.row, 'ID': coo.col, 'count': coo.data}\n",
    "                 )[['feature', 'ID', 'count']].sort_values(['feature', 'ID']\n",
    "                 ).reset_index(drop=True)\n",
    "\n",
    "mat.loc[:,'feature'] = mat.loc[:,'feature'].apply(lambda x: terms[x])\n",
    "print( mat.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.0 Load the transfusion related terms \n",
    "- specified by SMEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "xfus=pd.read_excel('terms_indicate_transfusion9.xlsx')\n",
    "\n",
    "xfus.columns=['ngrams']\n",
    "xfus['ngrams'] = xfus.ngrams.str.strip()\n",
    "xfus.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Find and Remove \n",
    "- remove the exact matches to the transfusion related terms specified by SMEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ngrams_orig = len(mat.feature.unique())\n",
    "print('num of ngrams=',ngrams_orig)\n",
    "\n",
    "data = ~mat['feature'].isin(list(xfus['ngrams'].values))\n",
    "print (data.value_counts())\n",
    "mat=mat[data]\n",
    "print( mat.shape)\n",
    "\n",
    "print('num of ngrams after removing transfused terms =',len(mat.feature.unique()))\n",
    "print('removed ' + str(ngrams_orig- len(mat.feature.unique())) + ' transfused terms')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 get IDs \n",
    "- to output the terms with the ratios, merge back to the 'terms' mat that can be imported at the top (on=feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "terms1=mat[~mat.feature.duplicated()]\n",
    "terms0.columns=['feature','ratio']\n",
    "terms2 = terms0.merge(terms1,how='right',on='feature')\n",
    "terms2 = terms2.drop(['ID','count'],axis=1).sort_values('ratio',axis=0)\n",
    "\n",
    "terms2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open('textfeatures_id.pickle','rb') as f:\n",
    "\n",
    "    ids=pickle.load(f, encoding='latin1')\n",
    "\n",
    "h=pd.read_sql('''select hadm_id from mimiciii.transfused_notes_unique;''',engine)\n",
    "\n",
    "mat.loc[:,'ID'] = mat.loc[:,'ID'].apply(lambda x: int(ids[x]))\n",
    "mat['hadm_id']=mat['ID']\n",
    "\n",
    "\n",
    "mat = pd.merge(mat, h, how='left',on='hadm_id')\n",
    "\n",
    "mat.ID = mat.hadm_id\n",
    "mat=mat.reset_index()\n",
    "mat.drop(['hadm_id','index'],axis=1, inplace=True)\n",
    "\n",
    "mat.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 Collapse to longest n-gram\n",
    " - find rows that are duplicates\n",
    " - see if those duplicates have a similar feature name (index)\n",
    "     - if yes, then save into **dupes** and **no_dupes** matricies\n",
    "     - input **dupes** to functions to collapse to longest feature name by removing shorter one (remove works b/c we are doing binary)\n",
    "      - put **no_dupes** back together with the de-duplicated **dupes** for completed matrix\n",
    "\n",
    "## 3.0 Transform the DataFrame\n",
    "- each document (admission) is now a col, and a one is present for each feature that belongs to that doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is pretty slow\n",
    "df = mat.pivot_table(index='feature',columns='ID',values=None).fillna(0).astype('int32').sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = df.columns.droplevel()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Find and Separate out n-grams with same patterns of occurrence \n",
    "\n",
    "- create a mat w/o any of these 'duplicates' \n",
    "- after keeping longest ngram, we will append the de-duped ones to this mat **no_dupes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data= df.duplicated(keep=False)\n",
    "\n",
    "no_dupes=df.loc[~data,:]\n",
    "\n",
    "print( no_dupes.shape)\n",
    "\n",
    "dupes=df.loc[data,:]\n",
    "\n",
    "ind=list(dupes.index)\n",
    "dupes.shape\n",
    "\n",
    "# grab the first of each duplicate for fuzzy matching below\n",
    "\n",
    "data1=dupes.duplicated()\n",
    "\n",
    "first=list(dupes.loc[~data1,:].index)\n",
    "len(first)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing the words that end up without a matching n-gram after colllapsing them.  \n",
    "first"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 keep longest n-gram\n",
    "- use fuzzy matching to select longest n-gram of the duplicates\n",
    "- results are in **dupes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def find_dupes(x):\n",
    "    if x.equals(dupes.loc[first[f],:]):\n",
    "        dind.append(x.name)\n",
    "\n",
    "def all_match(x):\n",
    "    return all(fuzz.partial_ratio(i,max(x, key=len)) ==100 for i in x)\n",
    "\n",
    "\n",
    "def some_match(x):\n",
    "    x=list(x)\n",
    "    return [i for i in x if fuzz.partial_ratio(i,max(x,key=len))==100], [i for i in x if fuzz.partial_ratio(i,max(x,key=len))!=100]\n",
    "    \n",
    "def remove(m, dind):\n",
    "    todrop=dind\n",
    "    for n in m:\n",
    "        del todrop[todrop.index(n)]\n",
    "    \n",
    "    dupes.drop(todrop, axis=0, inplace=True)\n",
    "    \n",
    "def rename(m, dind):\n",
    "    new_ind=m+dind[len(m):]\n",
    "    d = dict(zip(dind,new_ind))\n",
    "    print(d)\n",
    "    dupes.rename(index = d, inplace=True)\n",
    "    \n",
    "    dupes.drop(dind[len(m):], axis=0, inplace=True)\n",
    "    \n",
    "def duplicate_removal(match):\n",
    "    m=max(map(len, match))\n",
    "    m=[x for x in match if len(x) == m]\n",
    "    if len(m)==1:\n",
    "\n",
    "        remove(m,match)\n",
    "    else:\n",
    "        print( 'Error: Edge Case – Multiple longest terms')\n",
    "        print( m)\n",
    "        print (fuzz.partial_ratio(m[0],m[1]))\n",
    "    return m\n",
    "\n",
    "def recursive_match(match, unmatch):\n",
    "    if len(unmatch)>0 and len(match)>1:\n",
    "        l = duplicate_removal(match)\n",
    "        catch.append(l[0])\n",
    "        match, unmatch= some_match(unmatch)\n",
    "        return recursive_match(match, unmatch)\n",
    "    else:\n",
    "        return match, unmatch\n",
    "    \n",
    "def word_match(astring,bstring):\n",
    "    count=0\n",
    "    for a in astring.split(' '):\n",
    "        for b in bstring.split(' '):\n",
    "            if a==b:\n",
    "                count+=1\n",
    "    if count==0:\n",
    "        q.append(bstring)\n",
    "    \n",
    "\n",
    "def get_overlap(unmatch):\n",
    "    '''this is all based on length. it is cutting off the longer term and putting them together if one is shorter? '''\n",
    "    \n",
    "    a=unmatch[0]\n",
    "    for b in unmatch:\n",
    "        if a!=b:\n",
    "            d=SequenceMatcher(None, a, b)\n",
    "            pos_a, pos_b, size = d.find_longest_match(0,len(a),0,len(b))\n",
    "            if pos_a>pos_b and (pos_a==0 or pos_b==0) and size>0 and fuzz.partial_ratio(a,b)!=100:\n",
    "                a=a + ' ' + b\n",
    "            elif pos_b>pos_a and (pos_a==0 or pos_b==0) and size>0:\n",
    "                a=b + ' ' + a\n",
    "            elif pos_b==0 and pos_a == 0 and size>0:\n",
    "                a=max([a,b],key=len)\n",
    "            else:\n",
    "                word_match(a,b)\n",
    "    return a\n",
    "\n",
    "    \n",
    "dind=[]\n",
    "catch=[]\n",
    "q=[]\n",
    "new=[]\n",
    "\n",
    "for f in range(len(first)):\n",
    "    dupes.apply(find_dupes, axis=1)\n",
    "    if all_match(dind):\n",
    "        m = duplicate_removal(dind)\n",
    "\n",
    "        dind=[]\n",
    "    else:\n",
    "        match, unmatch=some_match(dind)\n",
    "        match,unmatch=recursive_match(match,unmatch)\n",
    "        \n",
    "        l=duplicate_removal(match)\n",
    "        unmatch=unmatch+l+catch\n",
    "        rows=unmatch\n",
    "\n",
    "        a = get_overlap(unmatch)\n",
    "        new.append(a)\n",
    "        if len(q)>0:\n",
    "            unmatch=q\n",
    "            q=[]\n",
    "            a=get_overlap(unmatch)\n",
    "            new.append(a)\n",
    "\n",
    "        if len(q)==1:\n",
    "            new.append(q[0])\n",
    "        while len(q)>1:\n",
    "            unmatch=q\n",
    "            q=[]\n",
    "            a=get_overlap(unmatch)\n",
    "            new.append(a)\n",
    "        if len(q)==1:\n",
    "            new.append(q[0])\n",
    "    \n",
    "        rename(new, rows)\n",
    "        \n",
    "        \n",
    "            \n",
    "        dind=[]\n",
    "        catch=[]\n",
    "        q=[]\n",
    "        new=[]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- note that there are similar ngrams here, it's b/c they don't have the same patterns of ocurrance. go look at original list to see what the matches were"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dupes.iloc[:,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- append our de-duped matrix to the matrix w/o dupes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out=dupes.append(no_dupes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 remove rows that sum to zero (just in case)\n",
    "- transpose, remove, transpose back"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out3=out.transpose()\n",
    "print(out.shape)\n",
    "\n",
    "out3=out3[~(out3.sum(axis=1)==0)]\n",
    "print(out3.shape)\n",
    "\n",
    "out4=out3.transpose()\n",
    "print(out4.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.4 see if there are any more duplicates left "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print (out4.duplicated(keep=False).value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path + path1 + 'NB_terms_ratio_all.pkl', 'rb') as f:\n",
    "\n",
    "    terms_all=pickle.load(f)\n",
    "    \n",
    "terms_all.rename(columns={'vocab': 'feature'},inplace=True)\n",
    "terms_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top = pd.DataFrame()\n",
    "top['feature']= out.index\n",
    "\n",
    "# merge with terms0 to go get the ratios for ordering \n",
    "topt = terms_all.merge(top, how='inner', on='feature', copy=True, indicator=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len(topt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# plot top terms  by ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_top_terms(df, fig_title, top_words):\n",
    "    fig = plt.figure(figsize=(8, 12),dpi=100)  \n",
    "    y_pos = np.arange(top_words)\n",
    "    \n",
    "    plt.barh(y_pos,df.ratio.tail(top_words), alpha=0.5)\n",
    "    plt.title(fig_title, fontsize=20)\n",
    "    plt.yticks(y_pos,df.feature.tail(top_words), fontsize=14)\n",
    "    plt.xlim(.70,.83)\n",
    "    #plt.suptitle('Key words', fontsize=16)\n",
    "    plt.xlabel('ratio', fontsize=20)\n",
    "    #plt.subplots_adjust(wspace=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop the 2 terms with nans\n",
    "top_p=  topt.dropna(axis=0, how='any')\n",
    "\n",
    "# strongest terms at top\n",
    "top_p.sort_values('ratio',ascending=False,inplace=True)\n",
    "plot_top_terms(top_p,'Top Terms by Ratio',50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_p.size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save results of these collapsed vocab terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_p.to_pickle('NB_5000_final.pkl')\n",
    "top_p.to_csv('NB_5000_final.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# look at distribution of the terms \n",
    "+ to see if there are a few terms that are present in all documents.\n",
    "+ if so, remove those terms "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plot_mat = out.merge(top_p,how='left',on='feature')\n",
    "plot_mat.sort_values(by='ratio',inplace=True)\n",
    "plot_mat.drop('ratio',axis=1,inplace=True)\n",
    "plot_mat.set_index('feature',drop=True,inplace=True)\n",
    "plot_mat.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## plot histogram of total frequency for each term"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_mat['total_count_freq'] = plot_mat.sum(axis=1)\n",
    "\n",
    "plot_mat['total_hadmids'] = plot_mat.astype(bool).sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plot_matt = plot_mat.drop('total')\n",
    "plot_mat.total_count_freq.plot.hist(logy=True,figsize=(20,15),use_index=True,bins=(100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_mat.total_hadmids.plot.hist(logy=True,figsize=(20,15),use_index=True,bins=(100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plot_mat.loc['total_words'] = 0\n",
    "plot_mat.loc['total_words'] = plot_mat.astype(bool).sum(axis=0)\n",
    "plot_mat.sort_values(by='total_words',axis=1,ascending=False,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plot_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Plot the sparsity pattern of arrays\n",
    "\"\"\"\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure, show\n",
    "import numpy as np\n",
    "\n",
    "fig = plt.figure(dpi=100, figsize=(30,30));\n",
    "\n",
    "ax4 = plt.gca() \n",
    "\n",
    "x = plot_mat.drop(columns=['total_count_freq','_merge'],axis=1)\n",
    "x.sort_values('total_hadmids',inplace=True, ascending=False)\n",
    "x.drop('total_hadmids',axis=1,inplace=True)\n",
    "x.drop('total_words',axis=0,inplace=True)\n",
    "\n",
    "ax4.spy(x, precision=0, markersize=.2, aspect='auto')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Plot the sparsity pattern of arrays\n",
    "\"\"\"\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure, show\n",
    "import numpy as np\n",
    "\n",
    "fig = plt.figure(dpi=100, figsize=(30,30));\n",
    "\n",
    "ax4 = plt.gca() \n",
    "\n",
    "\n",
    "xx = plot_mat.drop(columns=['_merge','total_hadmids'],axis=1)\n",
    "xx.sort_values('total_count_freq',ascending=False, inplace=True)\n",
    "xx.drop('total_count_freq',axis=1,inplace=True)\n",
    "xxt = xx.T\n",
    "xxt.drop('total_words',axis=1,inplace=True)\n",
    "\n",
    "ax4.spy(xxt, precision=0,markersize=.2, aspect='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "s = plot_mat.total_count_freq.sort_values()\n",
    "s = s.drop('total_words')\n",
    "s.tail(45).plot.barh(figsize=(30,50),fontsize=40,title='Terms by total count');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = plot_mat.total_hadmids.sort_values()\n",
    "s = s.drop('total_words')\n",
    "s.tail(45).plot.barh(figsize=(20,30),fontsize=28, title='Terms by number of transfused admissions');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# sort by the ratio\n",
    "plot_mat_out = plot_mat.merge(terms0,how='left',on='feature')\n",
    "plot_mat_out.sort_values(by='ratio',ascending=True,inplace=True)\n",
    "\n",
    "cols = list(plot_mat_out.columns.values)\n",
    "# reorder cols \n",
    "cols = cols[-1:] + cols[:-3]\n",
    "plot_mat_out = plot_mat_out[cols]\n",
    "\n",
    "plot_mat_out.set_index('feature',inplace=True,drop=True)\n",
    "#plot_mat_out.drop('total_words',axis=0,inplace=True)\n",
    "plot_mat_out.head(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "save the terms, ratio, and freq count for SMEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_mat_out1 = plot_mat_out.loc[:,['total_hadmids','total_count_freq','ratio']]\n",
    "plot_mat_out1.drop('total_words',axis=0,inplace=True)\n",
    "plot_mat_out1.to_csv(path +  'NB_top_'+ str(top_p.shape[0]) +'_terms_only_dist.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "now save terms, ratio, count, and all hadmids for SMEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_mat_out.to_csv(path + path1 + 'NB_top_'+ str(top_p.shape[0]) +'_hadmids_forSME.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn.commit();\n",
    "cur.close();\n",
    "conn.close();"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}