lawyer_notebook_wc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "%%capture\n",
    "from __future__ import print_function, division\n",
    "import urllib.request\n",
    "import appmode\n",
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import ipywidgets as widgets\n",
    "from ipywidgets import Output\n",
    "from ipywidgets import interact, interactive, fixed, interact_manual\n",
    "from ipywidgets import Button, Layout\n",
    "from IPython.display import Javascript\n",
    "import pandas as pd\n",
    "from IPython.display import display as display\n",
    "from ipywidgets import HBox, VBox\n",
    "from collections import OrderedDict\n",
    "from fuzzywuzzy import process \n",
    "from fuzzywuzzy import fuzz\n",
    "pd.options.display.html.table_schema = True\n",
    "pd.options.display.max_rows = None\n",
    "pd.options.display.max_columns = 10\n",
    "pd.options.display.max_colwidth = 90\n",
    "pd.options.display.width = 270\n",
    "pd.options.display.expand_frame_repr = True\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import LinearSVC\n",
    "from io import StringIO\n",
    "import csv\n",
    "import os\n",
    "import os.path\n",
    "import re\n",
    "from bs4 import BeautifulSoup as BeautifulSoup\n",
    "from openpyxl import Workbook as Workbook\n",
    "#import pixiedust\n",
    "#import pixiedust_node\n",
    "#import pixiedust_rosie\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>H&K Apps<span class=\"tocSkip\"></span></h1>\n",
    "<br></br>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#The-H&amp;K-Lawyer-Notebook\" data-toc-modified-id=\"The-H&amp;K-Lawyer-Notebook-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>The H&amp;K Lawyer Notebook</a></span><ul class=\"toc-item\"><li><span><a href=\"#Classification-Tool\" data-toc-modified-id=\"Classification-Tool-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Classification Tool</a></span></li><li><span><a href=\"#Data-Tool\" data-toc-modified-id=\"Data-Tool-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Data Tool</a></span></li><li><span><a href=\"#Bulk-Compare\" data-toc-modified-id=\"Bulk-Compare-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Bulk Compare</a></span></li><li><span><a href=\"#Fuzzy-Searching\" data-toc-modified-id=\"Fuzzy-Searching-1.4\"><span class=\"toc-item-num\">1.4&nbsp;&nbsp;</span>Fuzzy Searching</a></span></li><li><span><a href=\"#Find-Precedent\" data-toc-modified-id=\"Find-Precedent-1.5\"><span class=\"toc-item-num\">1.5&nbsp;&nbsp;</span>Find Precedent</a></span></li><li><span><a href=\"#QuickCAN-(Coming-Soon)\" data-toc-modified-id=\"QuickCAN-(Coming-Soon)-1.6\"><span class=\"toc-item-num\">1.6&nbsp;&nbsp;</span>QuickCAN <em>(Coming Soon)</em></a></span></li><li><span><a href=\"#QuickCAR-(Coming-Soon)\" data-toc-modified-id=\"QuickCAR-(Coming-Soon)-1.7\"><span class=\"toc-item-num\">1.7&nbsp;&nbsp;</span>QuickCAR <em>(Coming Soon)</em></a></span></li><li><span><a href=\"#Signature-Page-Generator\" data-toc-modified-id=\"Signature-Page-Generator-1.8\"><span class=\"toc-item-num\">1.8&nbsp;&nbsp;</span>Signature Page Generator</a></span></li><li><span><a href=\"#Closing-Binder-Generator\" data-toc-modified-id=\"Closing-Binder-Generator-1.9\"><span class=\"toc-item-num\">1.9&nbsp;&nbsp;</span>Closing Binder Generator</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_list_as_excel(path, clean_sent_list, raw_sent_list):\n",
    "    wb = Workbook()\n",
    "    ws = wb.active\n",
    "    column_cell_A = 'A'\n",
    "    column_cell_B = 'B'\n",
    "    column_cell_C = 'C'\n",
    "    ws[column_cell_A + str(1)] = 'text'\n",
    "    ws[column_cell_B + str(1)] = 'label'\n",
    "    ws[column_cell_C + str(1)] = 'raw'\n",
    "    list_len = len(clean_sent_list)\n",
    "    for i in range(0, list_len):\n",
    "        ws[column_cell_A + str(i + 2)] = clean_sent_list[i]\n",
    "        ws[column_cell_B + str(i + 2)] = 'label'\n",
    "        ws[column_cell_C + str(i + 2)] = raw_sent_list[i]\n",
    "    wb.save(path)\n",
    "    print('Excel write complete')\n",
    "\n",
    "\n",
    "def html_from_file_no_tags(file_path):\n",
    "    with open(file_path, 'rb') as myfile:\n",
    "        raw_text = myfile.read()\n",
    "    bsObj = BeautifulSoup(raw_text, 'lxml').text\n",
    "    return bsObj\n",
    "\n",
    "\n",
    "def normalize_sent(sent):\n",
    "    wpt = nltk.WordPunctTokenizer()\n",
    "    stop_words = nltk.corpus.stopwords.words('english')\n",
    "    sent = re.sub(r'[^a-zA-Z\\s]', '', sent, re.I | re.A)\n",
    "    sent = sent.strip()\n",
    "    tokens = wpt.tokenize(sent)\n",
    "    filtered_tokens = [token for token in tokens if token not in stop_words]\n",
    "    sent = ' '.join(filtered_tokens).lower()\n",
    "    return sent\n",
    "\n",
    "\n",
    "def html_from_file_no_tags(file_path):\n",
    "    with open(file_path, 'rb') as myfile:\n",
    "        raw_text = myfile.read()\n",
    "    bsObj = BeautifulSoup(raw_text, 'lxml').text\n",
    "    return bsObj\n",
    "\n",
    "\n",
    "def load_normal_with_stopwords(path):\n",
    "    doc = str(load_raw(path))\n",
    "    return normalize_document_return_list(doc)\n",
    "\n",
    "\n",
    "def load_normal_no_stopwords(path):\n",
    "    doc = load_raw(path)\n",
    "    norm = normalize_document_return_list(doc)\n",
    "    clean_sent = []\n",
    "    for sent in norm:\n",
    "        clean = remove_stop_words(sent)\n",
    "        clean_sent.append(clean)\n",
    "    return clean_sent\n",
    "\n",
    "\n",
    "def list_from_directory(path):\n",
    "    list_of_text = []\n",
    "    for file in os.listdir(path):\n",
    "        filename = os.fsdecode(file)\n",
    "        file_path = path + filename\n",
    "        text = load_raw(file_path)\n",
    "        text = str(text)\n",
    "        list_of_text.append(text)\n",
    "\n",
    "\n",
    "def load_excel(path):\n",
    "    print('finish')\n",
    "\n",
    "\n",
    "def load_list_from_csv(path):\n",
    "    with open(path, 'r') as f:\n",
    "        reader = csv.reader(f)\n",
    "        list_raw = list(reader)\n",
    "        list_clean = []\n",
    "        for i in list_raw:\n",
    "            x = ''.join(i)\n",
    "            list_clean.append(x)\n",
    "        list_clean_two = []\n",
    "        for i in list_clean:\n",
    "            x = ''.join(i)\n",
    "            list_clean_two.append(x)\n",
    "        return list_clean_two\n",
    "\n",
    "\n",
    "def load_raw(path, tags=False):\n",
    "    if path.endswith('html'):\n",
    "        if (tags):\n",
    "            return html_from_file_tags(path)\n",
    "        else:\n",
    "            return html_from_file_no_tags(path)\n",
    "    elif path.endswith('.txt'):\n",
    "        return str(text_from_file(path))\n",
    "    else:\n",
    "        try:\n",
    "            return str(text_from_binary(path))\n",
    "        except:\n",
    "            print('Failed to load as binary. Try reader that accepts url as argument (e.g., html_from_web_tags(url) or html_from_web_no_tags(url)).')\n",
    "\n",
    "\n",
    "def text_from_binary(file_path):\n",
    "    #text = textract.process(file_path, method='tesseract', language='eng')\n",
    "    text = 'Not set up for binary'\n",
    "    return text.decode('unicode_escape').encode('utf-8', 'ignore').strip()\n",
    "\n",
    "\n",
    "def html_from_file_tags(file_path):\n",
    "    with open(file_path, 'rb') as myfile:\n",
    "        raw_text = myfile.read()\n",
    "    return raw_text\n",
    "\n",
    "\n",
    "def html_from_web_no_tags(url):\n",
    "    response = urlopen(url)\n",
    "    bsObj = BeautifulSoup(response, 'lxml').text\n",
    "    return bsObj\n",
    "\n",
    "\n",
    "def html_from_web_tags(url):\n",
    "    response = urlopen(url)\n",
    "    tagged_text = response.read()\n",
    "    return tagged_text\n",
    "\n",
    "\n",
    "def text_from_file(file_path):\n",
    "    with open(file_path, 'rb') as myfile:\n",
    "        raw_text = myfile.read()\n",
    "    return raw_text\n",
    "\n",
    "\n",
    "def remove_stop_words(doc):\n",
    "    wpt = nltk.WordPunctTokenizer()\n",
    "    stop_words = nltk.corpus.stopwords.words('english')\n",
    "    tokens = wpt.tokenize(doc)\n",
    "    filtered_tokens = [token for token in tokens if token not in stop_words]\n",
    "    doc = ' '.join(filtered_tokens).lower()\n",
    "    return doc\n",
    "\n",
    "\n",
    "def normalize_document_return_list(doc):\n",
    "    wpt = nltk.WordPunctTokenizer()\n",
    "    stop_words = nltk.corpus.stopwords.words('english')\n",
    "    list_of_clean_sents = []\n",
    "    sent_list = tokenize.sent_tokenize(str(doc))\n",
    "    for sent in sent_list:\n",
    "        sent = re.sub(r'[^a-zA-Z\\s]', '', sent, re.I | re.A)\n",
    "        sent = sent.strip()\n",
    "        tokens = wpt.tokenize(sent)\n",
    "        filtered_tokens = [token for token in tokens if token not in stop_words]\n",
    "        sent = ' '.join(filtered_tokens).lower()\n",
    "        list_of_clean_sents.append(sent)\n",
    "    return list_of_clean_sents\n",
    "\n",
    "\n",
    "def tokenize_and_stem(text):\n",
    "    stemmer = SnowballStemmer(\"english\")\n",
    "    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]\n",
    "    filtered_tokens = []\n",
    "    for token in tokens:\n",
    "        if re.search('[a-zA-Z]', token):\n",
    "            filtered_tokens.append(token)\n",
    "    stems = [stemmer.stem(t) for t in filtered_tokens]\n",
    "    joined_text = \" \".join(stems)\n",
    "    print(joined_text)\n",
    "    return joined_text\n",
    "\n",
    "def create_progress_bar():\n",
    "    in_progress = widgets.IntProgress(\n",
    "    value=0,\n",
    "    min=0,\n",
    "    max=10,\n",
    "    step=1,\n",
    "    visible=False,\n",
    "    description='Status:',\n",
    "    bar_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    orientation='horizontal'\n",
    ")\n",
    "    return in_progress\n",
    "\n",
    "def change_progress_bar(name, percent_complete):\n",
    "    name.value = percent_complete\n",
    "\n",
    "def is_progress_bar_visible(name, visible):\n",
    "    name.visible = visible\n",
    "    \n",
    "def clear_output(name):\n",
    "    name.close\n",
    "    \n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from matplotlib import cm\n",
    "\n",
    "def plot3D():\n",
    "    X = np.arange(-5, 5, 0.25)\n",
    "    Y = np.arange(-5, 5, 0.25)\n",
    "    X, Y = np.meshgrid(X, Y)\n",
    "    R = np.sqrt(X**2 + Y**2)\n",
    "    Z = np.sin(R)\n",
    "    # Make the plot\n",
    "    fig = plt.figure()\n",
    "    ax = fig.gca(projection=\"3d\")\n",
    "    surf = ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap=cm.coolwarm,\n",
    "                       linewidth=0, antialiased=False)\n",
    "    ax.set_zlim(-1.01, 1.01)\n",
    "    fig.colorbar(surf, shrink=0.5, aspect=5)\n",
    "    plt.show()\n",
    "    \n",
    "def create_submit_button():\n",
    "    button = widgets.Button(\n",
    "    description='Submit',\n",
    "    disabled=False\n",
    "    )\n",
    "    button.style.button_color = 'steelblue'\n",
    "    button.layout={'border': '1px solid black'}\n",
    "    return button\n",
    "\n",
    "def create_download_button():\n",
    "    button = widgets.Button(\n",
    "    description='Download Excel',\n",
    "    disabled=False\n",
    "    )\n",
    "    button.style.button_color = 'Honeydew'\n",
    "    button.layout={'border': '1px solid black'}\n",
    "    return button\n",
    "\n",
    "def create_refresh_button():\n",
    "    button = widgets.Button(\n",
    "    description='Clear Output',\n",
    "    disabled=False\n",
    "    )\n",
    "    button.style.button_color = 'LightSlateGray'\n",
    "    button.layout={'border': '1px solid black'}\n",
    "    return button\n",
    "\n",
    "def create_chart_button():\n",
    "    button = widgets.Button(\n",
    "    description='Chart_It',\n",
    "    disabled=False\n",
    "    )\n",
    "    button.style.button_color = 'LightSkyBlue'\n",
    "    button.layout={'border': '1px solid black'}\n",
    "    return button\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt text](https://s3.amazonaws.com/blaze4/HK+logo.png)\n",
    "\n",
    "# The H&K Lawyer Notebook\n",
    "\n",
    "A collection of simple, yet powerful tools for lawyers to do their jobs better and more efficiently. By leveraging data analytics, natural language processing, machine learning, and other technology solutions, users are able to produce better quality work faster and e more efficient. Many of these tools are still under active development, so the H&K Lawyer Notebook is reserved for our more intrepid lawyers.\n",
    "\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification Tool\n",
    "\n",
    "Classify text using one of several trained machine learning models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d43f33b582464570932aeb8ff4a550f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7cbc2f84efec48b1ba24461525f3a7d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output(layout=Layout(border='.25px solid grey'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#TODO Create seperate lists for individual items for other subject matters\n",
    "#TODO Add training back-end\n",
    "\n",
    "results = ''\n",
    "out = widgets.Output(layout={'border': '.25px solid grey'})\n",
    "out_aux = widgets.Output()\n",
    "\n",
    "model_selection = widgets.RadioButtons(\n",
    "    options=['Google AutoML', 'SKLearn', 'TensorFlow', 'Keras', 'Gensim'],\n",
    "     value='SKLearn',\n",
    "    description='Select Platform:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "subject_matter_selection = widgets.RadioButtons(\n",
    "    options=['Litigation', 'Finance', 'Leasing', 'Private Equity', 'Corporate M&A', 'Bond Financing'],\n",
    "     value='Finance',\n",
    "    description='Practice Area:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "#subject_matter_selection.layout.width = '90px'\n",
    "\n",
    "file_path_text = widgets.Text(              \n",
    "    value='https://s3.amazonaws.com/blaze4/credit.txt',\n",
    "    placeholder='filename',\n",
    "    description='Filename:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "#if subject_matter_selection == 'Finance':\n",
    "clause_selection = widgets.SelectMultiple(\n",
    "    options=['All', 'MAC_definition', 'commitment_fee', 'change_control', 'change_control_prepayment', 'debt_issuance_prepayment', 'default_interest', 'disposition_assets_prepayment', 'equity_issuance_prepayment', 'eurodollar_rate', 'excess_cash_flow', 'fixed_charge_ratio', 'lender_inspection_rights', 'margin_leverage_ratio', 'margin_rating', 'monetary_default', 'net_worth', 'restricted_payments', 'Amendments_Consent', 'secured_facility', 'Amendments_Consent', 'unused_fee', 'voluntary_prepayment'],\n",
    "    value=['All'],\n",
    "    rows=10,\n",
    "    description='Clause selection:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "\n",
    "model_name = widgets.RadioButtons(\n",
    "    options=['SVM', 'LSTM', 'NB'],\n",
    "    value='SVM',\n",
    "    description='Model:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "save_file_path = widgets.Text(              \n",
    "    value='model.pickle',\n",
    "    placeholder='filename',\n",
    "    description='Path to Save:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "model_training_material = widgets.Text(              \n",
    "    value='training.xlsx',\n",
    "    placeholder='file path',\n",
    "    description='Train Docs:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "button = create_submit_button()\n",
    "download_button = create_download_button()\n",
    "refresh_button = create_refresh_button()\n",
    "tab1 = VBox(children=[HBox(children=[file_path_text])])\n",
    "tab2 = VBox(children=[HBox(children=[model_selection, subject_matter_selection, clause_selection])])\n",
    "tab3 = VBox(children=[HBox(children=[model_name, model_training_material, save_file_path])])\n",
    "tab = widgets.Tab(children=[tab2, tab1, tab3])\n",
    "tab.set_title(0, 'Configure')\n",
    "tab.set_title(1, 'Predict')\n",
    "tab.set_title(2, 'Training')\n",
    "HBox_classify = HBox(children=[button, refresh_button, download_button])\n",
    "VBox_classify = VBox(children=[tab, HBox_classify])\n",
    "VBox_classify.layout={'border': '.25px solid grey'}\n",
    "\n",
    "with out_aux:\n",
    "    display(VBox_classify)\n",
    "\n",
    "returned_values = {} \n",
    "\n",
    "def clear_output():\n",
    "    out.clear_output()\n",
    "    \n",
    "\n",
    "@download_button.on_click\n",
    "def download_button_clicked(b):\n",
    "    output = returned_values['df_output']\n",
    "    writer = pd.ExcelWriter('classification_results.xlsx')\n",
    "    output.to_excel(writer,'Sheet1')\n",
    "    writer.save()\n",
    "    download_button.description = 'Download complete'\n",
    "\n",
    "@refresh_button.on_click\n",
    "def refresh_button_clicked(b):\n",
    "    out.clear_output()\n",
    "    download_button.description = 'Download Excel'\n",
    "\n",
    "#@out.capture()\n",
    "@button.on_click\n",
    "def on_button_clicked(b):\n",
    "    clear_output()\n",
    "    download_button.description = 'Download Excel'\n",
    "    in_progress = create_progress_bar()\n",
    "    with out:\n",
    "        display(in_progress)\n",
    "    change_progress_bar(in_progress, 0)\n",
    "    #Load training corpus\n",
    "    #df = pd.read_excel('/home/ec2-user/SageMaker/data_b.xlsx') #Uncomment if running SageMaker\n",
    "    df = pd.read_excel('https://s3.amazonaws.com/blaze4/data_b.xlsx') #Uncomment if running local\n",
    "    df = df.sample(frac=1, axis=1).reset_index(drop=True)\n",
    "    df = df[pd.notnull(df['Text'])]\n",
    "    df = df.drop_duplicates()\n",
    "    change_progress_bar(in_progress, 2)\n",
    "    col = ['Label', 'Text']\n",
    "    df = df[col]\n",
    "    df.columns = ['Label', 'Text']\n",
    "    df = df.sample(frac=1).reset_index(drop=True)\n",
    "    #Vectorize and transform training corpus\n",
    "    df['category_id'] = df['Label'].factorize()[0]\n",
    "    change_progress_bar(in_progress, 4)\n",
    "    category_id_df = df[['Label', 'category_id']].drop_duplicates().sort_values('category_id')\n",
    "    category_to_id = dict(category_id_df.values)\n",
    "    id_to_category = dict(category_id_df[['category_id', 'Label']].values)\n",
    "    #Train model\n",
    "    X_train, X_test, y_train, y_test = train_test_split(df['Text'], df['Label'], random_state = 0)\n",
    "    count_vect = CountVectorizer()\n",
    "    X_train_counts = count_vect.fit_transform(X_train)\n",
    "    tfidf_transformer = TfidfTransformer()\n",
    "    X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)\n",
    "    clf = LinearSVC().fit(X_train_tfidf, y_train)\n",
    "    change_progress_bar(in_progress, 6)\n",
    "    #Load document to be reviewed, clause segment and normalize\n",
    "    current_raw_sent_list = []\n",
    "    clean_raw_list = []\n",
    "    current_clean_sent_list = []\n",
    "    with urllib.request.urlopen(file_path_text.value) as response:\n",
    "        text = response.read()\n",
    "    sent_list = sent_tokenize(str(text))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        current_clean_sent_list.append(clean_sent)\n",
    "        current_raw_sent_list.append(sent)\n",
    "    in_progress.value = 7\n",
    "    change_progress_bar(in_progress, 7)\n",
    "    for sent in current_raw_sent_list:\n",
    "        clean_raw = re.sub(\"\\xe2\", \" \", sent)\n",
    "        clean_raw = re.sub(\"\\x80\", \" \", clean_raw)\n",
    "        clean_raw = re.sub(r\"\\n\\n\", \" \", clean_raw)\n",
    "        clean_raw = re.sub(r'\\n', \" \", clean_raw)\n",
    "        clean_raw = re.sub(\"\\xe2\\x80\\x9c\", \" \", clean_raw)\n",
    "        clean_raw = re.sub(\"each\\nRevolving+\", \"each Revolving\", clean_raw)\n",
    "        clean_raw = re.sub(\"\\xe2\\x80\\x9cRequired\", \"Required\", clean_raw)\n",
    "        clean_raw = re.sub(\"\\xe2\\x80\\x9c\", ' ', clean_raw)\n",
    "        clean_raw = re.sub(\"\\xe2\\x80\\x99s\", ' ', clean_raw)\n",
    "        clean_raw = re.sub(\"\\sn\\s\", ' ', clean_raw)\n",
    "        clean_raw = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", clean_raw)\n",
    "        clean_raw_list.append(clean_raw)\n",
    "    #Load normalized sentences into dataframe\n",
    "    df_current = pd.DataFrame(columns=['label', 'raw', 'clean'])\n",
    "    df_current['clean'] = current_clean_sent_list\n",
    "    df_current['raw'] = clean_raw_list\n",
    "    change_progress_bar(in_progress, 8)\n",
    "    #Iterate through dataframe, predict clause type and record to [label] column in df\n",
    "    for index, row in df_current.iterrows():\n",
    "        test = []\n",
    "        test.append(df_current['clean'][index])\n",
    "        X_test=count_vect.transform(test)\n",
    "        z = clf.predict(X_test)\n",
    "        df_current['label'][index] = z[0]\n",
    "    change_progress_bar(in_progress, 10)\n",
    "    if clause_selection.value[0] == 'All':\n",
    "        clause_selection.value = ['MAC_definition', 'commitment_fee', 'change_control', 'change_control_prepayment', 'debt_issuance_prepayment', 'default_interest', 'disposition_assets_prepayment', 'equity_issuance_prepayment', 'eurodollar_rate', 'excess_cash_flow', 'fixed_charge_ratio', 'lender_inspection_rights', 'margin_leverage_ratio', 'margin_rating', 'monetary_default', 'net_worth', 'restricted_payments', 'Amendments_Consent', 'secured_facility', 'Amendments_Consent', 'unused_fee', 'voluntary_prepayment']\n",
    "    df_output = df_current[df_current.apply(lambda x: x['label'] in clause_selection.value, axis=1)]\n",
    "    returned_values['df_output'] = df_output \n",
    "    in_progress.close()\n",
    "    with out:\n",
    "        display(df_output)\n",
    "        \n",
    "        \n",
    "display(out_aux)\n",
    "out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Tool\n",
    "\n",
    "A tool for organizing and visualizing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#TODO design user input box; add charting; replace data set with car ride specific examples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7d13a2296ab43dfa0b2aa86d15c726a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94b0de6fe4c24c14ad7c742332558c7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_out = widgets.Output()\n",
    "data_UI_out = widgets.Output()\n",
    "\n",
    "\n",
    "chart_submit_button = create_chart_button()\n",
    "data_button = create_submit_button()\n",
    "data_download_button = create_download_button()\n",
    "data_refresh_button = create_refresh_button()\n",
    "\n",
    "@data_download_button.on_click\n",
    "def data_download_button_clicked(b):\n",
    "    output = returned_values['df_min']\n",
    "    writer = pd.ExcelWriter('/Users/josiasdewey/jupyter/notebooks/tmp/data_results.xlsx')\n",
    "    output.to_excel(writer,'Sheet1')\n",
    "    writer.save()\n",
    "    data_download_button.description = 'Download complete'\n",
    "\n",
    "@data_refresh_button.on_click\n",
    "def data_refresh_button_clicked(b):\n",
    "    data_out.clear_output()\n",
    "    \n",
    "def clear_output():\n",
    "    data_out.clear_output()\n",
    "\n",
    "@chart_submit_button.on_click\n",
    "#@data_out.capture(clear_output=True, wait=True)\n",
    "def chart_it(b):\n",
    "    c_in_progress = create_progress_bar()\n",
    "    display(c_in_progress)\n",
    "    change_progress_bar(c_in_progress, 2)\n",
    "    if data_selection.value == 'employee_set':\n",
    "        df = pd.read_csv('https://s3.amazonaws.com/blaze4/HR_comma_sep.csv')\n",
    "    else:\n",
    "        df = pd.read_csv('https://s3.amazonaws.com/blaze4/trip_stats_taz.csv')\n",
    "        #df = pd.read_csv('/Users/josiasdewey/jupyter/notebooks/static/trip_stats_taz.csv')\n",
    "    df_min = df[0:2000]\n",
    "    change_progress_bar(c_in_progress, 4)\n",
    "    if chart_selection.value == '3D Plot':\n",
    "        plot3D()\n",
    "        change_progress_bar(c_in_progress, 10)\n",
    "        c_in_progress.close()\n",
    "    else:\n",
    "        bp_x = df_min[data_points_selection_x.value]\n",
    "        bp_y = df_min[data_points_selection_y.value]\n",
    "        bp_z = df_min[data_points_selection_z.value]\n",
    "        if chart_selection.value == '3D Plot':\n",
    "            plot3D()\n",
    "        if chart_selection.value == 'Scatter Plot':\n",
    "            cht = plt.scatter(bp_x, bp_y,\n",
    "                 color=\"blue\", label=r\"Scatter\")\n",
    "        if chart_selection.value == 'Bar Graph':\n",
    "            cht = plt.bar(bp_x, bp_y,\n",
    "                 color=\"blue\", label=r\"Bar\")\n",
    "        if chart_selection.value == 'Bar Graph':\n",
    "            cht = plt.bar(bp_x, bp_y,\n",
    "                 color=\"blue\", label=r\"Bar\")\n",
    "        xlim_max = (1.25 * (df_min[data_points_selection_x.value].max()))\n",
    "        ylim_max = (1.25 * (df_min[data_points_selection_y.value].max()))\n",
    "        xlim_min = (-1.25 * (df_min[data_points_selection_x.value].min()))\n",
    "        ylim_min = (-1.25 * (df_min[data_points_selection_y.value].min()))\n",
    "        change_progress_bar(c_in_progress, 6)\n",
    "        plt.xlim(xlim_min, xlim_max)\n",
    "        plt.ylim(ylim_min, ylim_max)\n",
    "        plt.xlabel(data_points_selection_x.value)\n",
    "        plt.ylabel(data_points_selection_y.value)\n",
    "        plt.title(r\"Relationship between %s and %s\" % (data_points_selection_x.value, data_points_selection_y.value) )\n",
    "        fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "        fig_size[0] = 18\n",
    "        fig_size[1] = 9\n",
    "        plt.rcParams[\"figure.figsize\"] = fig_size\n",
    "        plt.legend(loc=\"lower left\")\n",
    "        change_progress_bar(c_in_progress, 8)\n",
    "        plt.ion()\n",
    "        plt.show()\n",
    "        change_progress_bar(c_in_progress, 10)\n",
    "        c_in_progress.close()\n",
    "        display(cht)\n",
    "    \n",
    "#@data_out.capture(clear_output=True, wait=True)\n",
    "@data_button.on_click\n",
    "def load_data(b):\n",
    "    clear_output()\n",
    "    d_in_progress = create_progress_bar()\n",
    "    display(d_in_progress)\n",
    "    change_progress_bar(d_in_progress, 2)\n",
    "    if data_selection.value == 'employee_set':\n",
    "        df = pd.read_csv('https://s3.amazonaws.com/blaze4/HR_comma_sep.csv')\n",
    "    else:\n",
    "        df = pd.read_csv('https://s3.amazonaws.com/blaze4/trip_stats_taz.csv')\n",
    "        #df = pd.read_csv('/Users/josiasdewey/jupyter/notebooks/static/trip_stats_taz.csv')\n",
    "    df_min = df[0:2000]\n",
    "    display(df_min)\n",
    "    change_progress_bar(d_in_progress, 4)\n",
    "    returned_values['df_data'] = df_min\n",
    "    change_progress_bar(d_in_progress, 8)\n",
    "    data_points_selection_x.options = list(df_min)\n",
    "    data_points_selection_y.options = list(df_min)\n",
    "    data_points_selection_z.options = list(df_min)\n",
    "    display(df_min)\n",
    "    change_progress_bar(d_in_progress, 10)\n",
    "    d_in_progress.close()\n",
    "    returned_values['df_min'] = df_min\n",
    "    with data_out:\n",
    "        display(df_min)\n",
    "    \n",
    "custom_file_path_text = widgets.Text(              \n",
    "    value='',\n",
    "    placeholder='file path',\n",
    "    description='Filename:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "data_selection = widgets.SelectMultiple(\n",
    "    options=['employee_set', 'ride_share_data_SF'],\n",
    "    value=['employee_set'],\n",
    "    rows=10,\n",
    "    description='Data Set:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "chart_selection = widgets.RadioButtons(\n",
    "    options=['Scatter Plot', '3D Plot', 'Bar Graph', 'Pie Graph', 'Line Graph'],\n",
    "    value='Scatter Plot',\n",
    "    description='Select chart:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "returned_values = {'df_data': ['load data', 'load data']}\n",
    "df = returned_values['df_data']\n",
    "df_col = list(df)\n",
    "data_points_selection_x = widgets.RadioButtons(\n",
    "    options=df_col,\n",
    "    value='load data',\n",
    "    description='x-axis:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "data_points_selection_y = widgets.RadioButtons(\n",
    "    options=df_col,\n",
    "    value='load data',\n",
    "    description='y-axis:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "data_points_selection_z = widgets.RadioButtons(\n",
    "    options=df_col,\n",
    "    value='load data',\n",
    "    description='z-axis:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "tab1 = VBox(children=[HBox(children=[data_selection, custom_file_path_text])])\n",
    "tab2 = VBox(children=[HBox(children=[chart_selection, \n",
    "                                     data_points_selection_x,\n",
    "                                     data_points_selection_y,\n",
    "                                     data_points_selection_z])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Data Sets')\n",
    "tab.set_title(1, 'Charting')\n",
    "HBox_data = HBox(children=[data_button, data_refresh_button, chart_submit_button, data_download_button])\n",
    "VBox_data = VBox(children=[tab, HBox_data])\n",
    "VBox_data.layout={'border': '.25px solid grey'}\n",
    "\n",
    "with data_UI_out:\n",
    "    display(VBox_data)\n",
    "    \n",
    "    \n",
    "display(data_UI_out)\n",
    "data_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bulk Compare\n",
    "\n",
    "Use this bulk comparison tool to compare a large number of documents against a common base form.  For example, compare lease agreements against a lease form to determine deviations from approved form. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_out = widgets.Output()\n",
    "compare_UI_out = widgets.Output()\n",
    "compare_button = create_submit_button()\n",
    "compare_refresh_button  = create_refresh_button()\n",
    "compare_download_button  = create_download_button()\n",
    "\n",
    "@compare_button.on_click\n",
    "def on_click(b):\n",
    "    doc_1_raw_sent_list = []\n",
    "    doc_1_clean_sent_list = []\n",
    "    sent_list = sent_tokenize(load_raw(file_path_text_1.value))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_1_clean_sent_list.append(clean_sent)\n",
    "        doc_1_raw_sent_list.append(sent)\n",
    "    df_doc_1 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_1['clean'] = doc_1_clean_sent_list\n",
    "    df_doc_1['raw'] = doc_1_raw_sent_list\n",
    "    \n",
    "    doc_2_raw_sent_list = []\n",
    "    doc_2_clean_sent_list = []\n",
    "    sent_list = sent_tokenize(load_raw(file_path_text_2.value))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_2_clean_sent_list.append(clean_sent)\n",
    "        doc_2_raw_sent_list.append(sent)\n",
    "    df_doc_2 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_2['clean'] = doc_2_clean_sent_list\n",
    "    df_doc_2['raw'] = doc_2_raw_sent_list\n",
    "    df_results = pd.DataFrame(columns=['query', 'match', 'strength'])\n",
    "    query = df_doc_1['clean']\n",
    "    choices = df_doc_2['clean']\n",
    "    query_list = []\n",
    "    found_list = []\n",
    "    strength_list = []\n",
    "    for q in query:\n",
    "        match = process.extractOne(q, choices=choices, scorer=fuzz.token_sort_ratio, score_cutoff=0)\n",
    "        if match != None:\n",
    "            index = match[2]\n",
    "            if match[1] >= compare_slider.value:\n",
    "                found = match[0]\n",
    "                strength = match[1]\n",
    "            else:\n",
    "                found = 'None'\n",
    "                strength = 'N/A'\n",
    "            query_list.append(q)\n",
    "            found_list.append(choices[index])\n",
    "            strength_list.append(strength)\n",
    "    df_results['query'] = query_list\n",
    "    df_results['found']= found_list\n",
    "    df_results['strength'] = strength_list\n",
    "    with compare_out:\n",
    "        display(df_results)\n",
    "\n",
    "file_path_text_1 = widgets.Text(\n",
    "    value='../static/Stock Purchase Agreement_original.txt',\n",
    "    placeholder='file path',\n",
    "    description='Base document:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "file_path_text_2 = widgets.Text(\n",
    "    value='',\n",
    "    placeholder='../static/Stock Purchase Agreement_new_deal.txt',\n",
    "    description='Docs to compare:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "compare_output_path = widgets.Text(\n",
    "    value='',\n",
    "    placeholder='file path for output',\n",
    "    description='output path:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "timeout_slider = widgets.FloatSlider(\n",
    "    value=0,\n",
    "    base=1,\n",
    "    min=0, # max exponent of base\n",
    "    max=1, # min exponent of base\n",
    "    step=0.1, # exponent step\n",
    "    description='Timeout:'\n",
    ")\n",
    "\n",
    "\n",
    "tab1 = VBox(children=[HBox(children=[file_path_text_1, file_path_text_2])])\n",
    "tab2 = VBox(children=[HBox(children=[timeout_slider])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Documents')\n",
    "tab.set_title(1, 'Settings')\n",
    "HBox_compare = HBox(children=[compare_button, compare_refresh_button, compare_download_button])\n",
    "VBox_compare = VBox(children=[tab, HBox_compare])\n",
    "with compare_UI_out:\n",
    "    display(VBox_compare)\n",
    "display(compare_UI_out)\n",
    "compare_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fuzzy Searching\n",
    "\n",
    "Use \"fuzzy\" searching to compare any number of items (the 'Query') against a large groups of possible matches (the 'Search'). For example, a client requests we confirm the destruction of files relating to several hundred matters, where the matter description for each must be cross checked against several thousand entries in our file storage list. This tool leverages fuzzy matching in order to identify matches undetected by pure booleen searches (e.g., \"FIRST INTERNATIONAL\" captured even if search query is \"INTL FIRS\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TO DO -- Finish VBox layout\n",
    "\n",
    "fuzzy_out = widgets.Output()\n",
    "fuzzy_UI_out = widgets.Output()\n",
    "\n",
    "fuzzy_button = create_submit_button()\n",
    "fuzzy_download_button = create_download_button()\n",
    "fuzzy_refresh_button = create_refresh_button()\n",
    "\n",
    "@fuzzy_button.on_click\n",
    "def on_click(b):\n",
    "    doc_1_raw_sent_list = []\n",
    "    doc_1_clean_sent_list = []\n",
    "    sent_list = sent_tokenize(load_raw(file_path_text_1.value))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_1_clean_sent_list.append(clean_sent)\n",
    "        doc_1_raw_sent_list.append(sent)\n",
    "    df_doc_1 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_1['clean'] = doc_1_clean_sent_list\n",
    "    df_doc_1['raw'] = doc_1_raw_sent_list\n",
    "    doc_2_raw_sent_list = []\n",
    "    doc_2_clean_sent_list = []\n",
    "    sent_list = sent_tokenize(load_raw(file_path_text_2.value))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_2_clean_sent_list.append(clean_sent)\n",
    "        doc_2_raw_sent_list.append(sent)\n",
    "    df_doc_2 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_2['clean'] = doc_2_clean_sent_list\n",
    "    df_doc_2['raw'] = doc_2_raw_sent_list\n",
    "    df_results = pd.DataFrame(columns=['query', 'match', 'strength'])\n",
    "    query = df_doc_1['clean']\n",
    "    choices = df_doc_2['clean']\n",
    "    query_list = []\n",
    "    found_list = []\n",
    "    strength_list = []\n",
    "    for q in query:\n",
    "        match = process.extractOne(q, choices=choices, scorer=fuzz.token_sort_ratio, score_cutoff=0)\n",
    "        if match != None:\n",
    "            index = match[2]\n",
    "            if match[1] >= compare_slider.value:\n",
    "                found = match[0]\n",
    "                strength = match[1]\n",
    "            else:\n",
    "                found = 'None'\n",
    "                strength = 'N/A'\n",
    "            query_list.append(q)\n",
    "            found_list.append(choices[index])\n",
    "            strength_list.append(strength)\n",
    "    df_results['query'] = query_list\n",
    "    df_results['found']= found_list\n",
    "    df_results['strength'] = strength_list\n",
    "    display(df_results)\n",
    "\n",
    "file_path_text_1 = widgets.Text(\n",
    "    value='query_set.xlsx',\n",
    "    placeholder='Type something',\n",
    "    description='Query:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "file_path_text_2 = widgets.Text(\n",
    "    value='search_set.xlsx',\n",
    "    placeholder='Type something',\n",
    "    description='Search:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "number_returned = widgets.Dropdown(\n",
    "    options=['1', '2', '3', '4'],\n",
    "    value='2',\n",
    "    description='Number returned:',\n",
    "    disabled=False,\n",
    ")\n",
    "\n",
    "t = widgets.Text(\n",
    "    value='filename and path',\n",
    "    placeholder='Type something',\n",
    "    description='Compare:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "fuzzy_slider = widgets.FloatSlider(\n",
    "    value=0,\n",
    "    base=100,\n",
    "    min=0, # max exponent of base\n",
    "    max=100, # min exponent of base\n",
    "    step=0.2, # exponent step\n",
    "    description='Similarity:'\n",
    ")\n",
    "\n",
    "tab1 = VBox(children=[HBox(children=[file_path_text_1, file_path_text_2])])\n",
    "tab2 = VBox(children=[HBox(children=[number_returned, fuzzy_slider])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Query Parameters')\n",
    "tab.set_title(1, 'Configure')\n",
    "HBox_fuzzy = HBox(children=[fuzzy_button, fuzzy_refresh_button, fuzzy_download_button])\n",
    "VBox_fuzzy = VBox(children=[tab, HBox_fuzzy])\n",
    "with fuzzy_UI_out:\n",
    "    display(VBox_fuzzy)\n",
    "    \n",
    "display(fuzzy_UI_out)\n",
    "fuzzy_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find Precedent\n",
    "Quickly search one or more curated clause banks for similar clauses.  Also, the user can identify form documents from which a particular agreememt originates.  The user can set a minimum similarity threshold to filter out less relevant clauses.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "precedent_out = widgets.Output()\n",
    "precedent_UI_out = widgets.Output()\n",
    "\n",
    "precedent_button = create_submit_button()\n",
    "precedent_download_button = create_download_button()\n",
    "precedent_refresh_button = create_refresh_button()\n",
    "precedent_compare_slider = widgets.FloatSlider(\n",
    "    value=0,\n",
    "    base=100,\n",
    "    min=0, # max exponent of base\n",
    "    max=100, # min exponent of base\n",
    "    step=0.2, # exponent step\n",
    "    description='Similarity:'\n",
    ")\n",
    "\n",
    "find_precedent_returned_values = {}\n",
    "\n",
    "@precedent_button.on_click\n",
    "def precedent_on_click(b):\n",
    "    doc_1_raw_sent_list = []\n",
    "    doc_1_clean_sent_list = []\n",
    "    sent_list = sent_tokenize(find_precedent_text_area.value)\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_1_clean_sent_list.append(clean_sent)\n",
    "        doc_1_raw_sent_list.append(sent)\n",
    "    df_doc_1 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_1['clean'] = doc_1_clean_sent_list\n",
    "    df_doc_1['raw'] = doc_1_raw_sent_list\n",
    "    df_precedent = pd.read_pickle('credit_pickle')\n",
    "    df_precedent2 = pd.read_pickle('/Users/josiasdewey/jupyter/Misc/d/credit_pickle2')\n",
    "    df_precedent = pd.concat([df_precedent, df_precedent2], axis=0)\n",
    "    df_precedent.to_pickle('master_credit_pickle')\n",
    "    df_results = pd.DataFrame(columns=['strength', 'query', 'match_raw'])\n",
    "    query = df_doc_1['clean']\n",
    "    choices = df_precedent['clean']\n",
    "    query_list = []\n",
    "    found_list = []\n",
    "    found_raw_list = []\n",
    "    strength_list = []\n",
    "    for q in query:\n",
    "        match = process.extract(q, choices=choices, scorer=fuzz.token_sort_ratio, limit=int(find_precedent_number_returned.value))\n",
    "        for tup in match:\n",
    "            if tup != None:\n",
    "                index = tup[2]\n",
    "                if tup[1] >= precedent_compare_slider.value:\n",
    "                    found = tup[0]\n",
    "                    found_raw = tup[0] #df_precedent['raw'][index]\n",
    "                    strength = tup[1]\n",
    "                    if q in query_list:\n",
    "                        query_list.append('')\n",
    "                    else:\n",
    "                        query_list.append(q)\n",
    "                    found_list.append(df_precedent['clean'][index])\n",
    "                    strength_list.append(strength)\n",
    "                    found_raw_list.append(found_raw)\n",
    "    df_results['query'] = query_list\n",
    "    #df_results['found']= found_list\n",
    "    df_results['strength'] = strength_list\n",
    "    df_results['match_raw'] = found_raw_list\n",
    "    find_precedent_returned_values['df_results'] = df_results \n",
    "    with precedent_out:\n",
    "        display(df_results)\n",
    "\n",
    "find_precedent_number_returned = widgets.Dropdown(\n",
    "    options=['1', '2', '3', '4', '5', '6','7', '8', '9', '10', '25', '50', '100'],\n",
    "    value='10',\n",
    "    description='Return:',\n",
    "    disabled=False,\n",
    ")\n",
    "    \n",
    "find_precedent_subject_matter_selection = widgets.RadioButtons(\n",
    "    options=['Litigation', 'Finance', 'Leasing', 'Private Equity', 'Corporate M&A', 'Bond Financing'],\n",
    "     value='Finance',\n",
    "    description='Area:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "find_precedent_text_area = widgets.Textarea(\n",
    "    value='Since February 1, 2014, there has been no material adverse change in the business, assets, operations or condition of Holdings and its Subsidiaries, taken as a whole.',\n",
    "    placeholder='',\n",
    "    description='Clause Text:',\n",
    "    width='500px',\n",
    "    height='50%',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "find_precedent_file_path_text = widgets.Text(              \n",
    "    value='credit.txt',\n",
    "    placeholder='Type something',\n",
    "    description='Filename:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "in_progress = widgets.IntProgress(\n",
    "    value=0,\n",
    "    min=0,\n",
    "    max=10,\n",
    "    step=1,\n",
    "    description='Loading:',\n",
    "    bar_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    orientation='horizontal'\n",
    ")\n",
    "\n",
    "@precedent_download_button.on_click\n",
    "def find_precedent_download_button_clicked(b):\n",
    "    output = find_precedent_returned_values['df_results']\n",
    "    writer = pd.ExcelWriter('classification_results.xlsx')\n",
    "    output.to_excel(writer,'Sheet1')\n",
    "    writer.save()\n",
    "    precedent_download_button.description = 'Download complete'\n",
    "\n",
    "refresh_button = widgets.Button(\n",
    "    description='Clear Output',\n",
    "    disabled=False,\n",
    "    button_style='info'\n",
    ")\n",
    "\n",
    "@precedent_refresh_button.on_click\n",
    "def find_precedent_refresh_button_clicked(b):\n",
    "    precedent_out.clear_output()\n",
    "\n",
    "\n",
    "tab1 = VBox(children=[HBox(children=[find_precedent_text_area, find_precedent_subject_matter_selection, find_precedent_number_returned, precedent_compare_slider])])\n",
    "tab2 = VBox(children=[HBox(children=[find_precedent_file_path_text, find_precedent_subject_matter_selection, find_precedent_number_returned, precedent_compare_slider])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Search by Clause')\n",
    "tab.set_title(1, 'Search by Document/File')\n",
    "HBox_precedent = HBox(children=[precedent_button, precedent_refresh_button, precedent_download_button])\n",
    "VBox_precedent = VBox(children=[tab, HBox_precedent])\n",
    "with precedent_UI_out:\n",
    "    display(VBox_precedent)\n",
    "\n",
    "display(precedent_UI_out)\n",
    "precedent_out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## QuickCAN *(Coming Soon)*\n",
    "Computer generated revisions to contracts based on precedent clause database.  Machine learning algorithms and fuzzy matching are used to incorporate revisions made to similar text in precedent clauses."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## QuickCAR *(Coming Soon)*\n",
    "\n",
    "Computer generated responses to requests for production and interrogetories. Machine learning algorithms and fuzzy matching are used to incorporate historical responses to similar requests.  Will automatically generate a first draft of a response."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Signature Page Generator\n",
    "***\n",
    "Generate signature page packets with the press of a button."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Closing Binder Generator\n",
    "***\n",
    "Automate the preparation of closing binders."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "binder_UI_out = widgets.Output()\n",
    "binder_out = widgets.Output()\n",
    "binder_button = create_submit_button()\n",
    "binder_download_button = create_download_button()\n",
    "binder_refresh_button = create_refresh_button()\n",
    "\n",
    "@binder_button.on_click\n",
    "def on_click(b):\n",
    "    pass\n",
    "\n",
    "binder_lender = widgets.RadioButtons(\n",
    "    options=['Wells Fargo', 'Bank of America (Private Wealth)', 'Bank of America', 'JPMorgan Chase', 'Ocean Bank', 'Other'],\n",
    "    value='Ocean Bank',\n",
    "    description='Lender:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "binder_type = widgets.RadioButtons(\n",
    "    options=['Middle Market', 'Construction Loan', 'Bank Loan', 'Loan Modification', 'ABL Loan', 'Art Loan'],\n",
    "    value='Bank Loan',\n",
    "    description='Loan Type:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "\n",
    "tab1 = VBox(children=[HBox(children=[binder_lender, binder_type])])\n",
    "tab2 = VBox(children=[HBox(children=[])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Loan Details')\n",
    "tab.set_title(1, 'Custom Templates')\n",
    "HBox_binder = HBox(children=[binder_button, binder_refresh_button, binder_download_button])\n",
    "VBox_binder = VBox(children=[tab, HBox_binder ])\n",
    "with binder_UI_out:\n",
    "    display(VBox_binder)\n",
    "    \n",
    "display(binder_UI_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": false,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "217.717px",
    "left": "38px",
    "top": "123.867px",
    "width": "261.283px"
   },
   "toc_section_display": false,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}