.lawyer_notebook-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "from __future__ import print_function, division\n",
    "import appmode\n",
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import ipywidgets as widgets\n",
    "from ipywidgets import Output\n",
    "from ipywidgets import interact, interactive, fixed, interact_manual\n",
    "from ipywidgets import Button, Layout\n",
    "from IPython.display import Javascript\n",
    "import pandas as pd\n",
    "from IPython.display import display as display\n",
    "from IPython.display import HTML\n",
    "from ipywidgets import HBox, VBox\n",
    "from collections import OrderedDict\n",
    "from fuzzywuzzy import process \n",
    "from fuzzywuzzy import fuzz\n",
    "pd.options.display.html.table_schema = True\n",
    "pd.options.display.max_rows = None\n",
    "pd.options.display.max_columns = 10\n",
    "pd.options.display.max_colwidth = 100\n",
    "pd.options.display.width = 300\n",
    "pd.options.display.expand_frame_repr = True\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import LinearSVC\n",
    "from io import StringIO\n",
    "import csv\n",
    "import os\n",
    "import os.path\n",
    "import re\n",
    "from bs4 import BeautifulSoup as BeautifulSoup\n",
    "from openpyxl import Workbook as Workbook\n",
    "from diff_match_patch import diff_match_patch\n",
    "#import pixiedust\n",
    "#import pixiedust_node\n",
    "#import pixiedust_rosie\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#The-H&amp;K-Lawyer-Notebook\" data-toc-modified-id=\"The-H&amp;K-Lawyer-Notebook-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>The H&amp;K Lawyer Notebook</a></span><ul class=\"toc-item\"><li><span><a href=\"#Classification-Tool\" data-toc-modified-id=\"Classification-Tool-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Classification Tool</a></span></li><li><span><a href=\"#Data-Tool\" data-toc-modified-id=\"Data-Tool-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Data Tool</a></span></li><li><span><a href=\"#Bulk-Compare\" data-toc-modified-id=\"Bulk-Compare-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Bulk Compare</a></span></li><li><span><a href=\"#Fuzzy-Searching\" data-toc-modified-id=\"Fuzzy-Searching-1.4\"><span class=\"toc-item-num\">1.4&nbsp;&nbsp;</span>Fuzzy Searching</a></span></li><li><span><a href=\"#Find-Precedent\" data-toc-modified-id=\"Find-Precedent-1.5\"><span class=\"toc-item-num\">1.5&nbsp;&nbsp;</span>Find Precedent</a></span></li><li><span><a href=\"#QuickCAN-(Coming-Soon)\" data-toc-modified-id=\"QuickCAN-(Coming-Soon)-1.6\"><span class=\"toc-item-num\">1.6&nbsp;&nbsp;</span>QuickCAN <em>(Coming Soon)</em></a></span></li><li><span><a href=\"#QuickCAR-(Coming-Soon)\" data-toc-modified-id=\"QuickCAR-(Coming-Soon)-1.7\"><span class=\"toc-item-num\">1.7&nbsp;&nbsp;</span>QuickCAR <em>(Coming Soon)</em></a></span></li><li><span><a href=\"#Signature-Page-Generator\" data-toc-modified-id=\"Signature-Page-Generator-1.8\"><span class=\"toc-item-num\">1.8&nbsp;&nbsp;</span>Signature Page Generator</a></span></li><li><span><a href=\"#Closing-Binder-Generator\" data-toc-modified-id=\"Closing-Binder-Generator-1.9\"><span class=\"toc-item-num\">1.9&nbsp;&nbsp;</span>Closing Binder Generator</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_list_as_excel(path, clean_sent_list, raw_sent_list):\n",
    "    wb = Workbook()\n",
    "    ws = wb.active\n",
    "    column_cell_A = 'A'\n",
    "    column_cell_B = 'B'\n",
    "    column_cell_C = 'C'\n",
    "    ws[column_cell_A + str(1)] = 'text'\n",
    "    ws[column_cell_B + str(1)] = 'label'\n",
    "    ws[column_cell_C + str(1)] = 'raw'\n",
    "    list_len = len(clean_sent_list)\n",
    "    for i in range(0, list_len):\n",
    "        ws[column_cell_A + str(i + 2)] = clean_sent_list[i]\n",
    "        ws[column_cell_B + str(i + 2)] = 'label'\n",
    "        ws[column_cell_C + str(i + 2)] = raw_sent_list[i]\n",
    "    wb.save(path)\n",
    "    print('Excel write complete')\n",
    "\n",
    "\n",
    "def html_from_file_no_tags(file_path):\n",
    "    with open(file_path, 'rb') as myfile:\n",
    "        raw_text = myfile.read()\n",
    "    bsObj = BeautifulSoup(raw_text, 'lxml').text\n",
    "    return bsObj\n",
    "\n",
    "\n",
    "def normalize_sent(sent):\n",
    "    wpt = nltk.WordPunctTokenizer()\n",
    "    stop_words = nltk.corpus.stopwords.words('english')\n",
    "    sent = re.sub(r'[^a-zA-Z\\s]', '', sent, re.I | re.A)\n",
    "    sent = sent.strip()\n",
    "    tokens = wpt.tokenize(sent)\n",
    "    filtered_tokens = [token for token in tokens if token not in stop_words]\n",
    "    sent = ' '.join(filtered_tokens).lower()\n",
    "    return sent\n",
    "\n",
    "\n",
    "def html_from_file_no_tags(file_path):\n",
    "    with open(file_path, 'rb') as myfile:\n",
    "        raw_text = myfile.read()\n",
    "    bsObj = BeautifulSoup(raw_text, 'lxml').text\n",
    "    return bsObj\n",
    "\n",
    "\n",
    "def load_normal_with_stopwords(path):\n",
    "    doc = str(load_raw(path))\n",
    "    return normalize_document_return_list(doc)\n",
    "\n",
    "\n",
    "def load_normal_no_stopwords(path):\n",
    "    doc = load_raw(path)\n",
    "    norm = normalize_document_return_list(doc)\n",
    "    clean_sent = []\n",
    "    for sent in norm:\n",
    "        clean = remove_stop_words(sent)\n",
    "        clean_sent.append(clean)\n",
    "    return clean_sent\n",
    "\n",
    "\n",
    "def list_from_directory(path):\n",
    "    list_of_text = []\n",
    "    for file in os.listdir(path):\n",
    "        filename = os.fsdecode(file)\n",
    "        file_path = path + filename\n",
    "        text = load_raw(file_path)\n",
    "        text = str(text)\n",
    "        list_of_text.append(text)\n",
    "\n",
    "\n",
    "def load_excel(path):\n",
    "    print('finish')\n",
    "\n",
    "\n",
    "def load_list_from_csv(path):\n",
    "    with open(path, 'r') as f:\n",
    "        reader = csv.reader(f)\n",
    "        list_raw = list(reader)\n",
    "        list_clean = []\n",
    "        for i in list_raw:\n",
    "            x = ''.join(i)\n",
    "            list_clean.append(x)\n",
    "        list_clean_two = []\n",
    "        for i in list_clean:\n",
    "            x = ''.join(i)\n",
    "            list_clean_two.append(x)\n",
    "        return list_clean_two\n",
    "\n",
    "\n",
    "def load_raw(path, tags=False):\n",
    "    if path.endswith('html'):\n",
    "        if (tags):\n",
    "            return html_from_file_tags(path)\n",
    "        else:\n",
    "            return html_from_file_no_tags(path)\n",
    "    elif path.endswith('.txt'):\n",
    "        return str(text_from_file(path))\n",
    "    else:\n",
    "        try:\n",
    "            return str(text_from_binary(path))\n",
    "        except:\n",
    "            print('Failed to load as binary. Try reader that accepts url as argument (e.g., html_from_web_tags(url) or html_from_web_no_tags(url)).')\n",
    "\n",
    "\n",
    "def text_from_binary(file_path):\n",
    "    #text = textract.process(file_path, method='tesseract', language='eng')\n",
    "    text = 'Not set up for binary'\n",
    "    return text.decode('unicode_escape').encode('utf-8', 'ignore').strip()\n",
    "\n",
    "\n",
    "def html_from_file_tags(file_path):\n",
    "    with open(file_path, 'rb') as myfile:\n",
    "        raw_text = myfile.read()\n",
    "    return raw_text\n",
    "\n",
    "\n",
    "def html_from_web_no_tags(url):\n",
    "    response = urlopen(url)\n",
    "    bsObj = BeautifulSoup(response, 'lxml').text\n",
    "    return bsObj\n",
    "\n",
    "\n",
    "def html_from_web_tags(url):\n",
    "    response = urlopen(url)\n",
    "    tagged_text = response.read()\n",
    "    return tagged_text\n",
    "\n",
    "\n",
    "def text_from_file(file_path):\n",
    "    with open(file_path, 'rb') as myfile:\n",
    "        raw_text = myfile.read()\n",
    "    return raw_text\n",
    "\n",
    "\n",
    "def remove_stop_words(doc):\n",
    "    wpt = nltk.WordPunctTokenizer()\n",
    "    stop_words = nltk.corpus.stopwords.words('english')\n",
    "    tokens = wpt.tokenize(doc)\n",
    "    filtered_tokens = [token for token in tokens if token not in stop_words]\n",
    "    doc = ' '.join(filtered_tokens).lower()\n",
    "    return doc\n",
    "\n",
    "\n",
    "def normalize_document_return_list(doc):\n",
    "    wpt = nltk.WordPunctTokenizer()\n",
    "    stop_words = nltk.corpus.stopwords.words('english')\n",
    "    list_of_clean_sents = []\n",
    "    sent_list = tokenize.sent_tokenize(str(doc))\n",
    "    for sent in sent_list:\n",
    "        sent = re.sub(r'[^a-zA-Z\\s]', '', sent, re.I | re.A)\n",
    "        sent = sent.strip()\n",
    "        tokens = wpt.tokenize(sent)\n",
    "        filtered_tokens = [token for token in tokens if token not in stop_words]\n",
    "        sent = ' '.join(filtered_tokens).lower()\n",
    "        list_of_clean_sents.append(sent)\n",
    "    return list_of_clean_sents\n",
    "\n",
    "\n",
    "def tokenize_and_stem(text):\n",
    "    stemmer = SnowballStemmer(\"english\")\n",
    "    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]\n",
    "    filtered_tokens = []\n",
    "    for token in tokens:\n",
    "        if re.search('[a-zA-Z]', token):\n",
    "            filtered_tokens.append(token)\n",
    "    stems = [stemmer.stem(t) for t in filtered_tokens]\n",
    "    joined_text = \" \".join(stems)\n",
    "    print(joined_text)\n",
    "    return joined_text\n",
    "\n",
    "def CleanAndSplitText(frame):\n",
    "\n",
    "    textDataOut = [] \n",
    "   \n",
    "    # This regular expression is for section headers in the bill summaries that we wish to ignore\n",
    "    reHeaders = re.compile(r\" *TABLE OF CONTENTS:? *\"\n",
    "                           \"| *Title [IVXLC]+:? *\"\n",
    "                           \"| *Section [IVXLC]+:? *\"\n",
    "                           \"| *Article [IVXLC]+:? *\"\n",
    "                           \"| *Subtitle [A-Z]+:? *\"\n",
    "                           \"| *\\(Sec\\. \\d+\\) *\")\n",
    "\n",
    "    # This regular expression is for punctuation that we wish to clean out\n",
    "    # We also will split sentences into smaller phrase like units using this expression\n",
    "    rePhraseBreaks = re.compile(\"[\\\"\\!\\?\\)\\]\\}\\,\\:\\;\\*\\-]*\\s+\\([0-9]+\\)\\s+[\\(\\[\\{\\\"\\*\\-]*\"                             \n",
    "                                \"|[\\\"\\!\\?\\)\\]\\}\\,\\:\\;\\*\\-]+\\s+[\\(\\[\\{\\\"\\*\\-]*\"\n",
    "                                \"|\\.\\.+\"\n",
    "                                \"|\\s*\\-\\-+\\s*\"\n",
    "                                \"|\\s+\\-\\s+\"\n",
    "                                \"|\\:\\:+\"\n",
    "                                \"|\\s+[\\/\\(\\[\\{\\\"\\-\\*]+\\s*\"\n",
    "                                \"|[\\,!\\?\\\"\\)\\(\\]\\[\\}\\{\\:\\;\\*](?=[a-zA-Z])\"\n",
    "                                \"|[\\\"\\!\\?\\)\\]\\}\\,\\:\\;]+[\\.]*$\"\n",
    "                             )\n",
    "    \n",
    "    # Regex for underbars\n",
    "    regexUnderbar = re.compile('_')\n",
    "    \n",
    "    # Regex for space\n",
    "    regexSpace = re.compile(' +')\n",
    " \n",
    "    # Regex for sentence final period\n",
    "    regexPeriod = re.compile(\"\\.$\")\n",
    "\n",
    "    # Iterate through each document and do:\n",
    "    #    (1) Split documents into sections based on section headers and remove section headers\n",
    "    #    (2) Split the sections into sentences using NLTK sentence tokenizer\n",
    "    #    (3) Further split sentences into phrasal units based on punctuation and remove punctuation\n",
    "    #    (4) Remove sentence final periods when not part of an abbreviation \n",
    "\n",
    "    for i in range(0, len(frame)):     \n",
    "        # Extract one document from frame\n",
    "        docID = frame['ID'][i]\n",
    "        docText = str(frame['Text'][i])\n",
    "\n",
    "        # Set counter for output line count for this document\n",
    "        lineIndex=0;\n",
    "\n",
    "        # Split document into sections by finding sections headers and splitting on them \n",
    "        sections = reHeaders.split(docText)\n",
    "        \n",
    "        for section in sections:\n",
    "            # Split section into sentence using NLTK tokenizer \n",
    "            sentences = tokenize.sent_tokenize(section)\n",
    "            \n",
    "            for sentence in sentences:\n",
    "                # Split each sentence into phrase level chunks based on punctuation\n",
    "                textSegs = rePhraseBreaks.split(sentence)\n",
    "                numSegs = len(textSegs)\n",
    "                \n",
    "                for j in range(0,numSegs):\n",
    "                    if len(textSegs[j])>0:\n",
    "                        # Convert underbars to spaces \n",
    "                        # Underbars are reserved for building the compound word phrases                   \n",
    "                        textSegs[j] = regexUnderbar.sub(\" \",textSegs[j])\n",
    "                    \n",
    "                        # Split out the words so we can specially handle the last word\n",
    "                        words = regexSpace.split(textSegs[j])\n",
    "                        phraseOut = \"\"\n",
    "                        # If the last word ends in a period then remove the period\n",
    "                        words[-1] = regexPeriod.sub(\"\", words[-1])\n",
    "                        # If the last word is an abbreviation like \"U.S.\"\n",
    "                        # then add the word final period back on\n",
    "                        if \"\\.\" in words[-1]:\n",
    "                            words[-1] += \".\"\n",
    "                        phraseOut = \" \".join(words)  \n",
    "\n",
    "                        textDataOut.append([docID, lineIndex, phraseOut])\n",
    "                        lineIndex += 1\n",
    "                        \n",
    "    # Convert to pandas frame \n",
    "    frameOut = pandas.DataFrame(textDataOut, columns=['DocID', 'DocLine', 'CleanedText'])                      \n",
    "    \n",
    "    return frameOut\n",
    "\n",
    "def create_progress_bar():\n",
    "    in_progress = widgets.IntProgress(\n",
    "    value=0,\n",
    "    min=0,\n",
    "    max=10,\n",
    "    step=1,\n",
    "    visible=False,\n",
    "    description='Status:',\n",
    "    bar_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    orientation='horizontal'\n",
    ")\n",
    "    return in_progress\n",
    "\n",
    "def change_progress_bar(name, percent_complete):\n",
    "    name.value = percent_complete\n",
    "\n",
    "def is_progress_bar_visible(name, visible):\n",
    "    name.visible = visible\n",
    "    \n",
    "def clear_output(name):\n",
    "    name.close\n",
    "    \n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from matplotlib import cm\n",
    "\n",
    "def plot3D():\n",
    "    X = np.arange(-5, 5, 0.25)\n",
    "    Y = np.arange(-5, 5, 0.25)\n",
    "    X, Y = np.meshgrid(X, Y)\n",
    "    R = np.sqrt(X**2 + Y**2)\n",
    "    Z = np.sin(R)\n",
    "    # Make the plot\n",
    "    fig = plt.figure()\n",
    "    ax = fig.gca(projection=\"3d\")\n",
    "    surf = ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap=cm.coolwarm,\n",
    "                       linewidth=0, antialiased=False)\n",
    "    ax.set_zlim(-1.01, 1.01)\n",
    "    fig.colorbar(surf, shrink=0.5, aspect=5)\n",
    "    plt.show()\n",
    "    \n",
    "def create_submit_button():\n",
    "    button = widgets.Button(\n",
    "    description='Submit',\n",
    "    disabled=False\n",
    "    )\n",
    "    button.style.button_color = 'steelblue'\n",
    "    button.layout={'border': '1px solid grey'}\n",
    "    return button\n",
    "\n",
    "def create_download_button():\n",
    "    button = widgets.Button(\n",
    "    description='Download Excel',\n",
    "    disabled=False\n",
    "    )\n",
    "    button.style.button_color = 'Honeydew'\n",
    "    button.layout={'border': '1px solid grey'}\n",
    "    return button\n",
    "\n",
    "def create_refresh_button():\n",
    "    button = widgets.Button(\n",
    "    description='Clear Output',\n",
    "    disabled=False\n",
    "    )\n",
    "    button.style.button_color = 'LightSlateGray'\n",
    "    button.layout={'border': '1px solid grey'}\n",
    "    return button\n",
    "\n",
    "def create_chart_button():\n",
    "    button = widgets.Button(\n",
    "    description='Chart_It',\n",
    "    disabled=False\n",
    "    )\n",
    "    button.style.button_color = 'LightSkyBlue'\n",
    "    button.layout={'border': '1px solid grey'}\n",
    "    return button\n",
    "\n",
    "\n",
    "def CleanAndSplitText(frame):\n",
    "\n",
    "    textDataOut = [] \n",
    "   \n",
    "    # This regular expression is for section headers in the bill summaries that we wish to ignore\n",
    "    reHeaders = re.compile(r\" *TABLE OF CONTENTS:? *\"\n",
    "                           \"| *Title [IVXLC]+:? *\"\n",
    "                           \"| *Section [IVXLC]+:? *\"\n",
    "                           \"| *Article [IVXLC]+:? *\"\n",
    "                           \"| *Subtitle [A-Z]+:? *\"\n",
    "                           \"| *\\(Sec\\. \\d+\\) *\")\n",
    "\n",
    "    # This regular expression is for punctuation that we wish to clean out\n",
    "    # We also will split sentences into smaller phrase like units using this expression\n",
    "    rePhraseBreaks = re.compile(\"[\\\"\\!\\?\\)\\]\\}\\,\\:\\;\\*\\-]*\\s+\\([0-9]+\\)\\s+[\\(\\[\\{\\\"\\*\\-]*\"                             \n",
    "                                \"|[\\\"\\!\\?\\)\\]\\}\\,\\:\\;\\*\\-]+\\s+[\\(\\[\\{\\\"\\*\\-]*\"\n",
    "                                \"|\\.\\.+\"\n",
    "                                \"|\\s*\\-\\-+\\s*\"\n",
    "                                \"|\\s+\\-\\s+\"\n",
    "                                \"|\\:\\:+\"\n",
    "                                \"|\\s+[\\/\\(\\[\\{\\\"\\-\\*]+\\s*\"\n",
    "                                \"|[\\,!\\?\\\"\\)\\(\\]\\[\\}\\{\\:\\;\\*](?=[a-zA-Z])\"\n",
    "                                \"|[\\\"\\!\\?\\)\\]\\}\\,\\:\\;]+[\\.]*$\"\n",
    "                             )\n",
    "    \n",
    "    # Regex for underbars\n",
    "    regexUnderbar = re.compile('_')\n",
    "    \n",
    "    # Regex for space\n",
    "    regexSpace = re.compile(' +')\n",
    " \n",
    "    # Regex for sentence final period\n",
    "    regexPeriod = re.compile(\"\\.$\")\n",
    "\n",
    "    # Iterate through each document and do:\n",
    "    #    (1) Split documents into sections based on section headers and remove section headers\n",
    "    #    (2) Split the sections into sentences using NLTK sentence tokenizer\n",
    "    #    (3) Further split sentences into phrasal units based on punctuation and remove punctuation\n",
    "    #    (4) Remove sentence final periods when not part of an abbreviation \n",
    "\n",
    "    for i in range(0, len(frame)):     \n",
    "        # Extract one document from frame\n",
    "        docID = frame['ID'][i]\n",
    "        docText = str(frame['Text'][i])\n",
    "\n",
    "        # Set counter for output line count for this document\n",
    "        lineIndex=0;\n",
    "\n",
    "        # Split document into sections by finding sections headers and splitting on them \n",
    "        sections = reHeaders.split(docText)\n",
    "        \n",
    "        for section in sections:\n",
    "            # Split section into sentence using NLTK tokenizer \n",
    "            sentences = tokenize.sent_tokenize(section)\n",
    "            \n",
    "            for sentence in sentences:\n",
    "                # Split each sentence into phrase level chunks based on punctuation\n",
    "                textSegs = rePhraseBreaks.split(sentence)\n",
    "                numSegs = len(textSegs)\n",
    "                \n",
    "                for j in range(0,numSegs):\n",
    "                    if len(textSegs[j])>0:\n",
    "                        # Convert underbars to spaces \n",
    "                        # Underbars are reserved for building the compound word phrases                   \n",
    "                        textSegs[j] = regexUnderbar.sub(\" \",textSegs[j])\n",
    "                    \n",
    "                        # Split out the words so we can specially handle the last word\n",
    "                        words = regexSpace.split(textSegs[j])\n",
    "                        phraseOut = \"\"\n",
    "                        # If the last word ends in a period then remove the period\n",
    "                        words[-1] = regexPeriod.sub(\"\", words[-1])\n",
    "                        # If the last word is an abbreviation like \"U.S.\"\n",
    "                        # then add the word final period back on\n",
    "                        if \"\\.\" in words[-1]:\n",
    "                            words[-1] += \".\"\n",
    "                        phraseOut = \" \".join(words)  \n",
    "\n",
    "                        textDataOut.append([docID, lineIndex, phraseOut])\n",
    "                        lineIndex += 1\n",
    "                        \n",
    "    # Convert to pandas frame \n",
    "    frameOut = pandas.DataFrame(textDataOut, columns=['DocID', 'DocLine', 'CleanedText'])                      \n",
    "    \n",
    "    return frameOut"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt text](https://s3.amazonaws.com/blaze4/HK+logo.png)\n",
    "\n",
    "# The H&K Lawyer Notebook\n",
    "\n",
    "A collection of simple, yet powerful tools for lawyers to do their jobs better and more efficiently. By leveraging data analytics, natural language processing, machine learning, and other technology solutions, users are able to produce better quality work faster and e more efficient. Many of these tools are still under active development, so the H&K Lawyer Notebook is reserved for our more intrepid lawyers.\n",
    "\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification Tool\n",
    "\n",
    "Classify text using one of several trained machine learning models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO Create seperate lists for individual items for other subject matters\n",
    "#TODO Add training back-end\n",
    "\n",
    "results = ''\n",
    "out = widgets.Output(layout={'border': '.25px solid grey'})\n",
    "out_aux = widgets.Output()\n",
    "\n",
    "model_selection = widgets.RadioButtons(\n",
    "    options=['Google AutoML', 'SKLearn', 'TensorFlow', 'Keras', 'Gensim'],\n",
    "     value='SKLearn',\n",
    "    description='Platform:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "subject_matter_selection = widgets.RadioButtons(\n",
    "    options=['Litigation', 'Finance', 'Leasing', 'Private Equity', 'Corporate M&A', 'Bond Financing'],\n",
    "     value='Finance',\n",
    "    description='Practice:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "file_path_text = widgets.Text(              \n",
    "    value='credit.txt',\n",
    "    placeholder='filename',\n",
    "    description='Filename:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "#if subject_matter_selection == 'Finance':\n",
    "clause_selection = widgets.SelectMultiple(\n",
    "    options=['All', 'MAC_definition', 'commitment_fee', 'change_control', 'change_control_prepayment', 'debt_issuance_prepayment', 'default_interest', 'disposition_assets_prepayment', 'equity_issuance_prepayment', 'eurodollar_rate', 'excess_cash_flow', 'fixed_charge_ratio', 'lender_inspection_rights', 'margin_leverage_ratio', 'margin_rating', 'monetary_default', 'net_worth', 'restricted_payments', 'Amendments_Consent', 'secured_facility', 'Amendments_Consent', 'unused_fee', 'voluntary_prepayment'],\n",
    "    value=['All'],\n",
    "    rows=10,\n",
    "    description='Clause selection:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "\n",
    "model_name = widgets.RadioButtons(\n",
    "    options=['SVM', 'LSTM', 'NB'],\n",
    "    value='SVM',\n",
    "    description='Model:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "save_file_path = widgets.Text(              \n",
    "    value='model.pickle',\n",
    "    placeholder='filename',\n",
    "    description='Path to Save:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "model_training_material = widgets.Text(              \n",
    "    value='training.xlsx',\n",
    "    placeholder='file path',\n",
    "    description='Train Docs:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "button = create_submit_button()\n",
    "download_button = create_download_button()\n",
    "refresh_button = create_refresh_button()\n",
    "tab1 = VBox(children=[HBox(children=[file_path_text])])\n",
    "tab2 = VBox(children=[HBox(children=[model_selection, subject_matter_selection, clause_selection])])\n",
    "tab3 = VBox(children=[HBox(children=[model_name, model_training_material, save_file_path])])\n",
    "tab = widgets.Tab(children=[tab2, tab1, tab3])\n",
    "tab.set_title(0, 'Configure')\n",
    "tab.set_title(1, 'Predict')\n",
    "tab.set_title(2, 'Training')\n",
    "HBox_classify = HBox(children=[button, refresh_button, download_button])\n",
    "VBox_classify = VBox(children=[tab, HBox_classify])\n",
    "VBox_classify.layout={'border': '.25px solid grey'}\n",
    "\n",
    "with out_aux:\n",
    "    display(VBox_classify)\n",
    "\n",
    "returned_values = {} \n",
    "\n",
    "def clear_output():\n",
    "    out.clear_output()\n",
    "    \n",
    "\n",
    "@download_button.on_click\n",
    "def download_button_clicked(b):\n",
    "    output = returned_values['df_output']\n",
    "    writer = pd.ExcelWriter('classification_results.xlsx')\n",
    "    output.to_excel(writer,'Sheet1')\n",
    "    writer.save()\n",
    "    download_button.description = 'Download complete'\n",
    "\n",
    "@refresh_button.on_click\n",
    "def refresh_button_clicked(b):\n",
    "    out.clear_output()\n",
    "    download_button.description = 'Download Excel'\n",
    "\n",
    "#@out.capture()\n",
    "@button.on_click\n",
    "def on_button_clicked(b):\n",
    "    clear_output()\n",
    "    download_button.description = 'Download Excel'\n",
    "    in_progress = create_progress_bar()\n",
    "    display(in_progress)\n",
    "    change_progress_bar(in_progress, 0)\n",
    "    #Load training corpus\n",
    "    #df = pd.read_excel('/home/ec2-user/SageMaker/data_b.xlsx') #Uncomment if running SageMaker\n",
    "    df = pd.read_excel('https://s3.amazonaws.com/blaze4/data_b.xlsx') #Uncomment if running local\n",
    "    df = df.sample(frac=1, axis=1).reset_index(drop=True)\n",
    "    df = df[pd.notnull(df['Text'])]\n",
    "    df = df.drop_duplicates()\n",
    "    change_progress_bar(in_progress, 2)\n",
    "    col = ['Label', 'Text']\n",
    "    df = df[col]\n",
    "    df.columns = ['Label', 'Text']\n",
    "    df = df.sample(frac=1).reset_index(drop=True)\n",
    "    #Vectorize and transform training corpus\n",
    "    df['category_id'] = df['Label'].factorize()[0]\n",
    "    change_progress_bar(in_progress, 4)\n",
    "    category_id_df = df[['Label', 'category_id']].drop_duplicates().sort_values('category_id')\n",
    "    category_to_id = dict(category_id_df.values)\n",
    "    id_to_category = dict(category_id_df[['category_id', 'Label']].values)\n",
    "    #Train model\n",
    "    X_train, X_test, y_train, y_test = train_test_split(df['Text'], df['Label'], random_state = 0)\n",
    "    count_vect = CountVectorizer()\n",
    "    X_train_counts = count_vect.fit_transform(X_train)\n",
    "    tfidf_transformer = TfidfTransformer()\n",
    "    X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)\n",
    "    clf = LinearSVC().fit(X_train_tfidf, y_train)\n",
    "    change_progress_bar(in_progress, 6)\n",
    "    #Load document to be reviewed, clause segment and normalize\n",
    "    current_raw_sent_list = []\n",
    "    clean_raw_list = []\n",
    "    current_clean_sent_list = []\n",
    "    sent_list = sent_tokenize(load_raw(file_path_text.value))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        current_clean_sent_list.append(clean_sent)\n",
    "        current_raw_sent_list.append(sent)\n",
    "    in_progress.value = 7\n",
    "    change_progress_bar(in_progress, 7)\n",
    "    for sent in current_raw_sent_list:\n",
    "        clean_raw = re.sub(\"\\xe2\", \" \", sent)\n",
    "        clean_raw = re.sub(\"\\x80\", \" \", clean_raw)\n",
    "        clean_raw = re.sub(r\"\\n\\n\", \" \", clean_raw)\n",
    "        clean_raw = re.sub(r'\\n', \" \", clean_raw)\n",
    "        clean_raw = re.sub(\"\\xe2\\x80\\x9c\", \" \", clean_raw)\n",
    "        clean_raw = re.sub(\"each\\nRevolving+\", \"each Revolving\", clean_raw)\n",
    "        clean_raw = re.sub(\"\\xe2\\x80\\x9cRequired\", \"Required\", clean_raw)\n",
    "        clean_raw = re.sub(\"\\xe2\\x80\\x9c\", ' ', clean_raw)\n",
    "        clean_raw = re.sub(\"\\xe2\\x80\\x99s\", ' ', clean_raw)\n",
    "        clean_raw = re.sub(\"\\sn\\s\", ' ', clean_raw)\n",
    "        clean_raw = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", clean_raw)\n",
    "        clean_raw_list.append(clean_raw)\n",
    "    #Load normalized sentences into dataframe\n",
    "    df_current = pd.DataFrame(columns=['label', 'raw', 'clean'])\n",
    "    df_current['clean'] = current_clean_sent_list\n",
    "    df_current['raw'] = clean_raw_list\n",
    "    change_progress_bar(in_progress, 8)\n",
    "    #Iterate through dataframe, predict clause type and record to [label] column in df\n",
    "    for index, row in df_current.iterrows():\n",
    "        test = []\n",
    "        test.append(df_current['clean'][index])\n",
    "        X_test=count_vect.transform(test)\n",
    "        z = clf.predict(X_test)\n",
    "        df_current['label'][index] = z[0]\n",
    "    change_progress_bar(in_progress, 10)\n",
    "    if clause_selection.value[0] == 'All':\n",
    "        clause_selection.value = ['MAC_definition', 'commitment_fee', 'change_control', 'change_control_prepayment', 'debt_issuance_prepayment', 'default_interest', 'disposition_assets_prepayment', 'equity_issuance_prepayment', 'eurodollar_rate', 'excess_cash_flow', 'fixed_charge_ratio', 'lender_inspection_rights', 'margin_leverage_ratio', 'margin_rating', 'monetary_default', 'net_worth', 'restricted_payments', 'Amendments_Consent', 'secured_facility', 'Amendments_Consent', 'unused_fee', 'voluntary_prepayment']\n",
    "    df_output = df_current[df_current.apply(lambda x: x['label'] in clause_selection.value, axis=1)]\n",
    "    returned_values['df_output'] = df_output \n",
    "    in_progress.close()                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    \n",
    "    with out:\n",
    "        display(df_output)\n",
    "display(out_aux)\n",
    "out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Tool\n",
    "\n",
    "A tool for organizing and visualizing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO design user input box; add charting; replace data set with car ride specific examples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_out = widgets.Output(layout={'border': '1px solid grey'})\n",
    "data_UI_out = widgets.Output()\n",
    "\n",
    "\n",
    "chart_submit_button = create_chart_button()\n",
    "data_button = create_submit_button()\n",
    "data_download_button = create_download_button()\n",
    "data_refresh_button = create_refresh_button()\n",
    "\n",
    "@data_download_button.on_click\n",
    "def data_download_button_clicked(b):\n",
    "    output = returned_values['df_min']\n",
    "    writer = pd.ExcelWriter('data_results.xlsx')\n",
    "    output.to_excel(writer,'Sheet1')\n",
    "    writer.save()\n",
    "    data_download_button.description = 'Download complete'\n",
    "\n",
    "def clear_output():\n",
    "    data_out.clear_output()\n",
    "    \n",
    "@data_refresh_button.on_click\n",
    "def data_refresh_button_clicked(b):\n",
    "    clear_output()\n",
    "\n",
    "@chart_submit_button.on_click\n",
    "#@data_out.capture(clear_output=True, wait=True)\n",
    "def chart_it(b):\n",
    "    c_in_progress = create_progress_bar()\n",
    "    display(c_in_progress)\n",
    "    change_progress_bar(c_in_progress, 2)\n",
    "    if data_selection.value == 'employee_set':\n",
    "        df = pd.read_csv('https://s3.amazonaws.com/blaze4/HR_comma_sep.csv')\n",
    "    else:\n",
    "        df = pd.read_csv('https://s3.amazonaws.com/blaze4/trip_stats_taz.csv')\n",
    "        #df = pd.read_csv('/Users/josiasdewey/jupyter/notebooks/static/trip_stats_taz.csv')\n",
    "    df_min = df[0:2000]\n",
    "    change_progress_bar(c_in_progress, 4)\n",
    "    if chart_selection.value == '3D Plot':\n",
    "        plot3D()\n",
    "        change_progress_bar(c_in_progress, 10)\n",
    "        c_in_progress.close()\n",
    "    else:\n",
    "        bp_x = df_min[data_points_selection_x.value]\n",
    "        bp_y = df_min[data_points_selection_y.value]\n",
    "        bp_z = df_min[data_points_selection_z.value]\n",
    "        if chart_selection.value == '3D Plot':\n",
    "            plot3D()\n",
    "        if chart_selection.value == 'Scatter Plot':\n",
    "            cht = plt.scatter(bp_x, bp_y,\n",
    "                 color=\"blue\", label=r\"Scatter\")\n",
    "        if chart_selection.value == 'Bar Graph':\n",
    "            cht = plt.bar(bp_x, bp_y,\n",
    "                 color=\"blue\", label=r\"Bar\")\n",
    "        if chart_selection.value == 'Bar Graph':\n",
    "            cht = plt.bar(bp_x, bp_y,\n",
    "                 color=\"blue\", label=r\"Bar\")\n",
    "        xlim_max = (1.25 * (df_min[data_points_selection_x.value].max()))\n",
    "        ylim_max = (1.25 * (df_min[data_points_selection_y.value].max()))\n",
    "        xlim_min = (-1.25 * (df_min[data_points_selection_x.value].min()))\n",
    "        ylim_min = (-1.25 * (df_min[data_points_selection_y.value].min()))\n",
    "        change_progress_bar(c_in_progress, 6)\n",
    "        plt.xlim(xlim_min, xlim_max)\n",
    "        plt.ylim(ylim_min, ylim_max)\n",
    "        plt.xlabel(data_points_selection_x.value)\n",
    "        plt.ylabel(data_points_selection_y.value)\n",
    "        plt.title(r\"Relationship between %s and %s\" % (data_points_selection_x.value, data_points_selection_y.value) )\n",
    "        fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "        fig_size[0] = 18\n",
    "        fig_size[1] = 9\n",
    "        plt.rcParams[\"figure.figsize\"] = fig_size\n",
    "        plt.legend(loc=\"lower left\")\n",
    "        change_progress_bar(c_in_progress, 8)\n",
    "        plt.ion()\n",
    "        plt.show()\n",
    "        change_progress_bar(c_in_progress, 10)\n",
    "        c_in_progress.close()\n",
    "        with data_out:\n",
    "            display(cht)\n",
    "    \n",
    "#@data_out.capture(clear_output=True, wait=True)\n",
    "@data_button.on_click\n",
    "def load_data(b):\n",
    "    clear_output()\n",
    "    d_in_progress = create_progress_bar()\n",
    "    display(d_in_progress)\n",
    "    change_progress_bar(d_in_progress, 2)\n",
    "    if data_selection.value == 'employee_set':\n",
    "        df = pd.read_csv('https://s3.amazonaws.com/blaze4/HR_comma_sep.csv')\n",
    "    else:\n",
    "        df = pd.read_csv('https://s3.amazonaws.com/blaze4/trip_stats_taz.csv')\n",
    "        #df = pd.read_csv('/Users/josiasdewey/jupyter/notebooks/static/trip_stats_taz.csv')\n",
    "    df_min = df[0:2000]\n",
    "    change_progress_bar(d_in_progress, 4)\n",
    "    returned_values['df_data'] = df_min\n",
    "    change_progress_bar(d_in_progress, 8)\n",
    "    data_points_selection_x.options = list(df_min)\n",
    "    data_points_selection_y.options = list(df_min)\n",
    "    data_points_selection_z.options = list(df_min)\n",
    "    change_progress_bar(d_in_progress, 10)\n",
    "    d_in_progress.close()\n",
    "    returned_values['df_min'] = df_min\n",
    "    with data_out:\n",
    "        display(df_min)\n",
    "    \n",
    "custom_file_path_text = widgets.Text(              \n",
    "    value='',\n",
    "    placeholder='file path',\n",
    "    description='Filename:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "data_selection = widgets.SelectMultiple(\n",
    "    options=['employee_set', 'ride_share_data_SF'],\n",
    "    value=['employee_set'],\n",
    "    rows=10,\n",
    "    description='Data Set:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "chart_selection = widgets.RadioButtons(\n",
    "    options=['Scatter Plot', '3D Plot', 'Bar Graph', 'Pie Graph', 'Line Graph'],\n",
    "    value='Scatter Plot',\n",
    "    description='Select chart:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "returned_values = {'df_data': ['load data', 'load data']}\n",
    "df = returned_values['df_data']\n",
    "df_col = list(df)\n",
    "data_points_selection_x = widgets.RadioButtons(\n",
    "    options=df_col,\n",
    "    value='load data',\n",
    "    description='x-axis:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "data_points_selection_y = widgets.RadioButtons(\n",
    "    options=df_col,\n",
    "    value='load data',\n",
    "    description='y-axis:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "data_points_selection_z = widgets.RadioButtons(\n",
    "    options=df_col,\n",
    "    value='load data',\n",
    "    description='z-axis:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "tab1 = VBox(children=[HBox(children=[data_selection, custom_file_path_text])])\n",
    "tab2 = VBox(children=[HBox(children=[chart_selection, \n",
    "                                     data_points_selection_x,\n",
    "                                     data_points_selection_y,\n",
    "                                     data_points_selection_z])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Data Sets')\n",
    "tab.set_title(1, 'Charting')\n",
    "HBox_data = HBox(children=[data_button, data_refresh_button, chart_submit_button, data_download_button])\n",
    "VBox_data = VBox(children=[tab, HBox_data])\n",
    "VBox_data.layout={'border': '.25px solid grey'}\n",
    "with data_UI_out:\n",
    "    display(VBox_data)\n",
    "    \n",
    "display(data_UI_out)\n",
    "data_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bulk Compare\n",
    "\n",
    "Use this bulk comparison tool to compare a large number of documents against a common base form.  For example, compare lease agreements against a lease form to determine deviations from approved form. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_out = widgets.Output(layout={'border': '.25px solid grey'}\n",
    ")\n",
    "compare_UI_out = widgets.Output()\n",
    "compare_button = create_submit_button()\n",
    "compare_refresh_button  = create_refresh_button()\n",
    "compare_download_button  = create_download_button()\n",
    "\n",
    "@compare_button.on_click\n",
    "def on_click(b):\n",
    "    doc_1_raw_sent_list = []\n",
    "    doc_1_clean_sent_list = []\n",
    "    base_raw = text_from_file(compare_file_path_text_1.value)\n",
    "    sent_list = sent_tokenize(str(base_raw))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_1_clean_sent_list.append(clean_sent)\n",
    "        doc_1_raw_sent_list.append(sent)\n",
    "    df_doc_1 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_1['clean'] = doc_1_clean_sent_list\n",
    "    df_doc_1['raw'] = doc_1_raw_sent_list\n",
    "    \n",
    "    doc_2_raw_sent_list = []\n",
    "    doc_2_clean_sent_list = []\n",
    "    compare_raw = text_from_file(compare_file_path_text_2.value)\n",
    "    sent_list = sent_tokenize(str(compare_raw))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_2_clean_sent_list.append(clean_sent)\n",
    "        doc_2_raw_sent_list.append(sent)\n",
    "    df_doc_2 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_2['clean'] = doc_2_clean_sent_list\n",
    "    df_doc_2['raw'] = doc_2_raw_sent_list\n",
    "    df_results = pd.DataFrame(columns=['query', 'match', 'strength', 'compare'])\n",
    "    query = df_doc_1['clean']\n",
    "    choices = df_doc_2['clean']\n",
    "    query_list = []\n",
    "    found_list = []\n",
    "    strength_list = []\n",
    "    for q in query:\n",
    "        match = process.extractOne(q, choices=choices, scorer=fuzz.token_sort_ratio, score_cutoff=0)\n",
    "        if match != None:\n",
    "            index = match[2]\n",
    "            found = match[0]\n",
    "            strength = match[1]\n",
    "            query_list.append(q)\n",
    "            found_list.append(choices[index])\n",
    "            strength_list.append(strength)\n",
    "    df_results['query'] = query_list\n",
    "    df_results['found']= found_list\n",
    "    df_results['strength'] = strength_list\n",
    "    #precedent = df_master[\"Result\"] \n",
    "    #current = df_master[\"Query\"] \n",
    "    compared_file = []\n",
    "    compared_clause = [] \n",
    "    clean_clause = []\n",
    "    dmp = diff_match_patch()\n",
    "    dmp.Diff_Timeout = 0\n",
    "    for index, row in df_results.iterrows():\n",
    "        diff = dmp.diff_main(str(row['found']), str(row['query']))\n",
    "        view = dmp.diff_prettyHtml(diff) \n",
    "        #display(HTML(view))\n",
    "        compared_file.append(view)\n",
    "        joined = ' '.join(compared_file)\n",
    "    with open('test.html', 'w+') as f:\n",
    "        f.write(joined)\n",
    "    df_results['compare'] = compared_file\n",
    "    with compare_out:\n",
    "        #display(HTML(df_results.to_html()))\n",
    "    #return df_results\n",
    "        html_1 = \"\"\"\n",
    "<!DOCTYPE html>\n",
    "<html>\n",
    "<head>\n",
    "<style>\n",
    "table {\n",
    "    font-family: arial, sans-serif;\n",
    "    border-collapse: collapse;\n",
    "    width: 100%;\n",
    "}\n",
    "\n",
    "td, th {\n",
    "    border: 1px solid #dddddd;\n",
    "    text-align: left;\n",
    "    padding: 8px;\n",
    "}\n",
    "\n",
    "tr:nth-child(even) {\n",
    "    background-color: #dddddd;\n",
    "}\n",
    "</style>\n",
    "</head>\n",
    "<body>\n",
    "\n",
    "<h2>Comparison Table</h2>\n",
    "\n",
    "<table>\n",
    "  <tr>\n",
    "    <th>Current</th>\n",
    "    <th>Proposed</th>\n",
    "    <th>Comparison</th>\n",
    "  </tr>\n",
    "    \"\"\"\n",
    "        html_rows = []\n",
    "        rows =len (compared_file)\n",
    "        for i in range(rows):\n",
    "            html_row = \"<tr>\" + \"<th>current</th>\" + \"<th>proposed</th>\"+ \"<th>\" + str(compared_file[i]) + \"</th></tr>\"\n",
    "            html_rows.append(html_row)\n",
    "        html_rows.append('</table></body></html>')\n",
    "    html_2 = ''.join(compared_file)\n",
    "    html = html_1 + html_2\n",
    "    display(HTML(html))\n",
    "\n",
    "compare_file_path_text_1 = widgets.Text(\n",
    "    value='Stock Purchase Agreement_original.txt',\n",
    "    placeholder='file path',\n",
    "    description='Base document:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "compare_file_path_text_2 = widgets.Text(\n",
    "    value='Stock Purchase Agreement_new_deal.txt',\n",
    "    placeholder='',\n",
    "    description='Docs to compare:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "compare_output_path = widgets.Text(\n",
    "    value='',\n",
    "    placeholder='file path for output',\n",
    "    description='output path:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "timeout_slider = widgets.FloatSlider(\n",
    "    value=0,\n",
    "    base=1,\n",
    "    min=0, # max exponent of base\n",
    "    max=1, # min exponent of base\n",
    "    step=0.1, # exponent step\n",
    "    description='Timeout:'\n",
    ")\n",
    "\n",
    "\n",
    "tab1 = VBox(children=[HBox(children=[compare_file_path_text_1, compare_file_path_text_2])])\n",
    "tab2 = VBox(children=[HBox(children=[timeout_slider])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Documents')\n",
    "tab.set_title(1, 'Settings')\n",
    "HBox_compare = HBox(children=[compare_button, compare_refresh_button, compare_download_button])\n",
    "VBox_compare = VBox(children=[tab, HBox_compare])\n",
    "VBox_compare.layout={'border': '.25px solid grey'}\n",
    "\n",
    "with compare_UI_out:\n",
    "    display(VBox_compare)\n",
    "display(compare_UI_out)\n",
    "compare_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fuzzy Searching\n",
    "\n",
    "Use \"fuzzy\" searching to compare any number of items (the 'Query') against a large groups of possible matches (the 'Search'). For example, a client requests we confirm the destruction of files relating to several hundred matters, where the matter description for each must be cross checked against several thousand entries in our file storage list. This tool leverages fuzzy matching in order to identify matches undetected by pure booleen searches (e.g., \"FIRST INTERNATIONAL\" captured even if search query is \"INTL FIRS\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "caa4c4ad72324998ac8b7c973617706e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b07593f0b32b4d1ebf845046e9153778",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output(layout=Layout(border='.25px solid grey'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#TO DO -- Finish VBox layout\n",
    "\n",
    "fuzzy_out = widgets.Output(layout={'border': '.25px solid grey'}\n",
    ")\n",
    "fuzzy_UI_out = widgets.Output()\n",
    "\n",
    "fuzzy_button = create_submit_button()\n",
    "fuzzy_download_button = create_download_button()\n",
    "fuzzy_refresh_button = create_refresh_button()\n",
    "\n",
    "@fuzzy_button.on_click\n",
    "def on_click(b):\n",
    "    doc_1_raw_sent_list = []\n",
    "    doc_1_clean_sent_list = []\n",
    "    doc_1_raw = load_raw(file_path_text_1.value)\n",
    "    sent_list = sent_tokenize(str(doc_1_raw))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_1_clean_sent_list.append(clean_sent)\n",
    "        doc_1_raw_sent_list.append(sent)\n",
    "    df_doc_1 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_1['clean'] = doc_1_clean_sent_list\n",
    "    df_doc_1['raw'] = doc_1_raw_sent_list\n",
    "    doc_2_raw_sent_list = []\n",
    "    doc_2_clean_sent_list = []\n",
    "    doc2_raw = load_raw(file_path_text_2.value)\n",
    "    sent_list = sent_tokenize(str(doc2_raw))\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_2_clean_sent_list.append(clean_sent)\n",
    "        doc_2_raw_sent_list.append(sent)\n",
    "    df_doc_2 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_2['clean'] = doc_2_clean_sent_list\n",
    "    df_doc_2['raw'] = doc_2_raw_sent_list\n",
    "    df_results = pd.DataFrame(columns=['query', 'match', 'strength'])\n",
    "    query = df_doc_1['clean']\n",
    "    choices = df_doc_2['clean']\n",
    "    query_list = []\n",
    "    found_list = []\n",
    "    strength_list = []\n",
    "    for q in query:\n",
    "        match = process.extractOne(q, choices=choices, scorer=fuzz.token_sort_ratio, score_cutoff=0)\n",
    "        if match != None:\n",
    "            index = match[2]\n",
    "            if match[1] >= fuzzy_slider.value:\n",
    "                found = match[0]\n",
    "                strength = match[1]\n",
    "            else:\n",
    "                found = 'None'\n",
    "                strength = 'N/A'\n",
    "            query_list.append(q)\n",
    "            found_list.append(choices[index])\n",
    "            strength_list.append(strength)\n",
    "    df_results['query'] = query_list\n",
    "    df_results['found']= found_list\n",
    "    df_results['strength'] = strength_list\n",
    "    display(df_results)\n",
    "    \n",
    "\n",
    "file_path_text_1 = widgets.Text(\n",
    "    value='query_set.xlsx',\n",
    "    placeholder='Type something',\n",
    "    description='Query:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "file_path_text_2 = widgets.Text(\n",
    "    value='search_set.xlsx',\n",
    "    placeholder='Type something',\n",
    "    description='Search:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "number_returned = widgets.Dropdown(\n",
    "    options=['1', '2', '3', '4'],\n",
    "    value='2',\n",
    "    description='Number returned:',\n",
    "    disabled=False,\n",
    ")\n",
    "\n",
    "t = widgets.Text(\n",
    "    value='filename and path',\n",
    "    placeholder='Type something',\n",
    "    description='Compare:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "fuzzy_slider = widgets.FloatSlider(\n",
    "    value=0,\n",
    "    base=100,\n",
    "    min=0, # max exponent of base\n",
    "    max=100, # min exponent of base\n",
    "    step=0.2, # exponent step\n",
    "    description='Similarity:'\n",
    ")\n",
    "\n",
    "tab1 = VBox(children=[HBox(children=[file_path_text_1, file_path_text_2])])\n",
    "tab2 = VBox(children=[HBox(children=[number_returned, fuzzy_slider])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Query Parameters')\n",
    "tab.set_title(1, 'Configure')\n",
    "HBox_fuzzy = HBox(children=[fuzzy_button, fuzzy_refresh_button, fuzzy_download_button])\n",
    "VBox_fuzzy = VBox(children=[tab, HBox_fuzzy])\n",
    "VBox_fuzzy.layout={'border': '.25px solid grey'}\n",
    "\n",
    "with fuzzy_UI_out:\n",
    "    display(VBox_fuzzy)\n",
    "    \n",
    "display(fuzzy_UI_out)\n",
    "fuzzy_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find Precedent\n",
    "Quickly search one or more curated clause banks for similar clauses.  Also, the user can identify form documents from which a particular agreememt originates.  The user can set a minimum similarity threshold to filter out less relevant clauses.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad1f034efd8d4432b8c4ec01b0fd6332",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e427d4d2884a49d4bc88835dd7f83361",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output(layout=Layout(border='.25px solid grey'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "precedent_out = widgets.Output(layout={'border': '.25px solid grey'})\n",
    "precedent_UI_out = widgets.Output()\n",
    "\n",
    "precedent_button = create_submit_button()\n",
    "precedent_download_button = create_download_button()\n",
    "precedent_refresh_button = create_refresh_button()\n",
    "\n",
    "\n",
    "find_precedent_returned_values = {}\n",
    "html_template = \"\"\"\n",
    "<!DOCTYPE html>\n",
    "<html>\n",
    "<head>\n",
    "<style>\n",
    "table {\n",
    "    font-family: arial, sans-serif;\n",
    "    border-collapse: collapse;\n",
    "    width: 100%;\n",
    "}\n",
    "\n",
    "td, th {\n",
    "    border: 1px solid #dddddd;\n",
    "    text-align: left;\n",
    "    padding: 8px;\n",
    "}\n",
    "\n",
    "tr:nth-child(even) {\n",
    "    background-color: #dddddd;\n",
    "}\n",
    "</style>\n",
    "</head>\n",
    "<body>\n",
    "\n",
    "<h2>Comparison Table</h2>\n",
    "\n",
    "<table>\n",
    "  <tr>\n",
    "    <th>Current/Strength</th>\n",
    "    <th>Alternative</th>\n",
    "    <th>Comparison</th>\n",
    "  </tr>\n",
    "\"\"\"\n",
    "\n",
    "def generate_html():\n",
    "    html = [html_template]\n",
    "    df_iter = pd.read_excel('/Users/josiasdewey/jupyter/notebooks/test.xlsx')\n",
    "    for index, row in df_iter.iterrows():\n",
    "        if index == 0:\n",
    "            html_row = \"<tr><th>\"+str(row['query_raw'])+\"</th><th>\"+'----'+\"</th><th>\" + '----' + \"</th></tr>\"\n",
    "        else:\n",
    "            html_row = \"<tr><th>\"+str(row['strength'])+\"</th><th>\"+str(row['match_raw'])+\"</th><th>\" + str(row['compare']) + \"</th></tr>\"\n",
    "        html.append(html_row)\n",
    "    html.append('</table></body></html>')\n",
    "    html = ''.join(html)\n",
    "    return html\n",
    "\n",
    "@precedent_button.on_click\n",
    "def precedent_on_click(b):\n",
    "    clear_output()\n",
    "    doc_1_raw_sent_list = []\n",
    "    doc_1_clean_sent_list = []\n",
    "    sent_list = sent_tokenize(find_precedent_text_area.value)\n",
    "    for sent in sent_list:\n",
    "        clean_sent = normalize_sent(sent)\n",
    "        doc_1_clean_sent_list.append(clean_sent)\n",
    "        doc_1_raw_sent_list.append(sent)\n",
    "    df_doc_1 = pd.DataFrame(columns=['raw', 'clean'])\n",
    "    df_doc_1['clean'] = doc_1_clean_sent_list\n",
    "    df_doc_1['raw'] = doc_1_raw_sent_list\n",
    "    #df_precedent = pd.read_pickle('credit_pickle')\n",
    "    #df_precedent2 = pd.read_pickle('/Users/josiasdewey/jupyter/Misc/d/credit_pickle2')\n",
    "    #df_precedent = pd.concat([df_precedent, df_precedent2], axis=0)\n",
    "    if find_precedent_subject_matter_selection.value == 'Finance':\n",
    "        df_precedent = pd.read_pickle('master_credit_pickle')\n",
    "    elif find_precedent_subject_matter_selection.value == 'Real Estate':\n",
    "        #df_precedent = pd.read_pickle('master_dirt_pickle')\n",
    "        df_precedent = pd.read_excel('master_dirt_excel.xlsx')\n",
    "    df_results = pd.DataFrame(columns=['strength', 'query', 'query_raw', 'match_raw', 'compare'])\n",
    "    query = (df_doc_1['clean'], df_doc_1['raw'])\n",
    "    choices = df_precedent['clean']\n",
    "    query_list = []\n",
    "    found_list = []\n",
    "    found_raw_list = []\n",
    "    strength_list = []\n",
    "    query_raw_list = []\n",
    "    for q in query[0]:\n",
    "        match = process.extract(q, choices=choices, scorer=fuzz.token_sort_ratio, limit=int(find_precedent_number_returned.value))\n",
    "        for tup in match:\n",
    "            if tup != None:\n",
    "                index = tup[2]\n",
    "                if tup[1] >= precedent_compare_slider.value:\n",
    "                    found = tup[0]\n",
    "                    found_raw = df_precedent['raw_1'][index]\n",
    "                    strength = tup[1]\n",
    "                    if q in query_list:\n",
    "                        query_list.append('')\n",
    "                        query_raw_list.append('')\n",
    "                    else:\n",
    "                        query_list.append(q)\n",
    "                        query_raw_list.append(df_doc_1['raw'][0])\n",
    "                        found_list.append(df_precedent['clean'][index])\n",
    "                    strength_list.append(strength)\n",
    "                    found_raw_list.append(found_raw)\n",
    "    compared_file = []\n",
    "    clean_clause = []\n",
    "    dmp = diff_match_patch()\n",
    "    dmp.Diff_Timeout = 0\n",
    "    for i in range(len(found_raw_list)):\n",
    "        diff = dmp.diff_main(str(query[1]), str(found_raw_list[i]))\n",
    "        view = dmp.diff_prettyHtml(diff) \n",
    "        compared_file.append(view)\n",
    "    df_results['compare'] = compared_file\n",
    "    df_results['query'] = query_list\n",
    "    df_results['strength'] = strength_list\n",
    "    df_results['match_raw'] = found_raw_list\n",
    "    df_results['query_raw'] = query_raw_list\n",
    "    find_precedent_returned_values['df_results'] = df_results\n",
    "    df_results.to_excel('test.xlsx')\n",
    "    html = generate_html()\n",
    "    with precedent_out:\n",
    "        display(HTML(html))\n",
    "        #display(df_results)\n",
    "\n",
    "\n",
    "find_precedent_number_returned = widgets.Dropdown(\n",
    "    options=['1', '2', '3', '4', '5', '6','7', '8', '9', '10', '25', '50', '100'],\n",
    "    value='10',\n",
    "    description='Return:',\n",
    "    disabled=False,\n",
    ")\n",
    "\n",
    "precedent_compare_slider = widgets.FloatSlider(\n",
    "    value=0,\n",
    "    base=100,\n",
    "    min=0, # max exponent of base\n",
    "    max=100, # min exponent of base\n",
    "    step=0.2, # exponent step\n",
    "    description='Similarity:'\n",
    ")\n",
    "precedent_compare_slider.layout = {'height' : '100px', 'width' : '400px', 'positioning': 'left'}\n",
    "\n",
    "\n",
    "find_precedent_number_returned.layout = {'padding' : '15px'} #{'height' : '25px', 'width' : '150px', 'positioning': 'left'}\n",
    "\n",
    "    \n",
    "find_precedent_subject_matter_selection = widgets.RadioButtons(\n",
    "    options=['Finance', 'Litigation', 'Real Estate','Leasing', 'Private Equity', 'Corporate M&A', 'Bond Financing'],\n",
    "     value='Finance',\n",
    "    description='Practice:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "#find_precedent_subject_matter_selection.layout = {'height' : '25px', 'width' : '300px', 'positioning': 'left'}\n",
    "find_precedent_subject_matter_selection.layout = {'padding' : '15px'} #, 'width' : '300px', 'positioning': 'left'}\n",
    "\n",
    "\n",
    "find_precedent_text_area = widgets.Textarea(\n",
    "    value='Since February 1, 2014, there has been no material adverse change in the business, assets, operations or condition of Holdings and its Subsidiaries, taken as a whole.',\n",
    "    placeholder='',\n",
    "    description='Clause Text:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "find_precedent_text_area.layout = {'height' : '300px', 'width' : '275px'}\n",
    "\n",
    "find_precedent_file_path_text = widgets.Text(              \n",
    "    value='credit.txt',\n",
    "    placeholder='Type something',\n",
    "    description='Filename:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "in_progress = widgets.IntProgress(\n",
    "    value=0,\n",
    "    min=0,\n",
    "    max=10,\n",
    "    step=1,\n",
    "    description='Loading:',\n",
    "    bar_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    orientation='horizontal'\n",
    ")\n",
    "\n",
    "\n",
    "@precedent_download_button.on_click\n",
    "def find_precedent_download_button_clicked(b):\n",
    "    output = find_precedent_returned_values['df_results']\n",
    "    writer = pd.ExcelWriter('classification_results.xlsx')\n",
    "    output.to_excel(writer,'Sheet1')\n",
    "    writer.save()\n",
    "    precedent_download_button.description = 'Download complete'\n",
    "\n",
    "def clear_output():\n",
    "    precedent_out.clear_output()\n",
    "\n",
    "@precedent_refresh_button.on_click\n",
    "def find_precedent_refresh_button_clicked(b):\n",
    "    precedent_out.clear_output()\n",
    "\n",
    "VBox_thing = VBox(children=[find_precedent_subject_matter_selection,find_precedent_number_returned, precedent_compare_slider])\n",
    "tab1 = VBox(children=[HBox(children=[find_precedent_text_area, VBox_thing])])\n",
    "tab2 = VBox(children=[HBox(children=[find_precedent_file_path_text, VBox_thing])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Search by Clause')\n",
    "tab.set_title(1, 'Search by Document/File')\n",
    "HBox_precedent = HBox(children=[precedent_button, precedent_refresh_button, precedent_download_button])\n",
    "VBox_precedent = VBox(children=[tab, HBox_precedent])\n",
    "VBox_precedent.layout={'border': '.25px solid grey'}\n",
    "\n",
    "with precedent_UI_out:\n",
    "    display(VBox_precedent)\n",
    "\n",
    "display(precedent_UI_out)\n",
    "precedent_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## QuickCAN *(Coming Soon)*\n",
    "Computer generated revisions to contracts based on precedent clause database.  Machine learning algorithms and fuzzy matching are used to incorporate revisions made to similar text in precedent clauses."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## QuickCAR *(Coming Soon)*\n",
    "\n",
    "Computer generated responses to requests for production and interrogetories. Machine learning algorithms and fuzzy matching are used to incorporate historical responses to similar requests.  Will automatically generate a first draft of a response."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Signature Page Generator\n",
    "***\n",
    "Generate signature page packets with the press of a button."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Closing Binder Generator\n",
    "***\n",
    "Automate the preparation of closing binders."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af06377cbfd240fe8b1ba86f88bc6cd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output(layout=Layout(border='.25px solid grey'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "binder_UI_out = widgets.Output(layout={'border': '.25px solid grey'})\n",
    "binder_out = widgets.Output()\n",
    "binder_button = create_submit_button()\n",
    "binder_download_button = create_download_button()\n",
    "binder_refresh_button = create_refresh_button()\n",
    "\n",
    "@binder_button.on_click\n",
    "def on_click(b):\n",
    "    pass\n",
    "\n",
    "binder_lender = widgets.RadioButtons(\n",
    "    options=['Wells Fargo', 'Bank of America (Private Wealth)', 'Bank of America', 'JPMorgan Chase', 'Ocean Bank', 'Other'],\n",
    "    value='Ocean Bank',\n",
    "    description='Lender:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "binder_type = widgets.RadioButtons(\n",
    "    options=['Middle Market', 'Construction Loan', 'Bank Loan', 'Loan Modification', 'ABL Loan', 'Art Loan'],\n",
    "    value='Bank Loan',\n",
    "    description='Loan Type:',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "\n",
    "tab1 = VBox(children=[HBox(children=[binder_lender, binder_type])])\n",
    "tab2 = VBox(children=[HBox(children=[])])\n",
    "tab = widgets.Tab(children=[tab1, tab2])\n",
    "tab.set_title(0, 'Loan Details')\n",
    "tab.set_title(1, 'Custom Templates')\n",
    "HBox_binder = HBox(children=[binder_button, binder_refresh_button, binder_download_button])\n",
    "VBox_binder = VBox(children=[tab, HBox_binder ])\n",
    "VBox_binder.layout={'border': '.25px solid grey'}\n",
    "with binder_UI_out:\n",
    "    display(VBox_binder)\n",
    "    \n",
    "display(binder_UI_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": false,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "217.717px",
    "left": "38px",
    "top": "123.867px",
    "width": "261.283px"
   },
   "toc_section_display": false,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}