Operator abnormality detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from time import time\n",
    "import transliterate\n",
    "from pymystem3 import Mystem\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "from tqdm import tqdm_notebook\n",
    "from multiprocessing import Pool, cpu_count\n",
    "from joblib import Parallel, delayed\n",
    "import pickle\n",
    "\n",
    "from training_utils import perplexity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import LambdaCallback\n",
    "from keras.models import Sequential\n",
    "from keras.preprocessing import sequence\n",
    "from keras.layers import Dense, Activation, GlobalAveragePooling1D, Dropout, Embedding\n",
    "from keras.layers import LSTM\n",
    "from keras.optimizers import RMSprop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2785: DtypeWarning: Columns (12) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('dialogs_1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('extracted_stopwords.txt','r') as f:\n",
    "    ru_stopwords = map(lambda s: s.replace('\\n',''), f.readlines())\n",
    "    ru_stopwords = set(ru_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import transliterate\n",
    "from pymystem3 import Mystem\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "\n",
    "pattern = r'''(?x)          # set flag to allow verbose regexps\n",
    "        (?:[A-Z]\\.)+        # abbreviations, e.g. U.S.A.\n",
    "      | \\w+(?:-\\w+)*        # words with optional internal hyphens\n",
    "      | \\$?\\d+(?:\\.\\d+)?%?  # currency and percentages, e.g. $12.40, 82%\n",
    "    '''\n",
    "# tokenizer = RegexpTokenizer('\\w+|\\$[\\d\\.]+|\\S+')\n",
    "tokenizer = RegexpTokenizer(pattern)\n",
    "\n",
    "mystem = Mystem()\n",
    "def clean_and_normalize_text(text):\n",
    "    text = re.sub(r'<.*?>', ' ', text)\n",
    "    text = text.replace('x', ' ')\n",
    "    text = mystem.lemmatize(text)\n",
    "    text = tokenizer.tokenize(' '.join(text))\n",
    "    text = map(lambda word: transliterate.translit(word,'ru'), text)\n",
    "    text = filter(lambda w: w not in ru_stopwords,text)\n",
    "    return ' '.join(list(text))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_sentences(st):\n",
    "    sentences = re.split(r'[.?!]\\s*', st)\n",
    "    if sentences[-1]:\n",
    "        return sentences\n",
    "    else:\n",
    "        return sentences[:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract and clean support responses "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "support_responses = df[df['Тип'] =='1-ответ']['Контент'].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpora = []\n",
    "for resp in support_responses:\n",
    "    corpora.extend(split_sentences(resp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c093643eef044dc181b23fd2a37ebe49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=520602), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pool = Pool(cpu_count())\n",
    "corpora = [resp for resp in tqdm_notebook(pool.imap_unordered(clean_and_normalize_text,corpora), total=len(corpora))]\n",
    "pool.close()\n",
    "pool.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0fb0138670f495d8ca92ff6c5ee9cb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=520602), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "word_count = Counter()\n",
    "for resp in tqdm_notebook(corpora):\n",
    "    for word in resp.split():\n",
    "        word_count[word] +=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f9324839828>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGJ1JREFUeJzt3XlwnHed5/H3ty+1Tsu25PiQD9lxSJzgHJg4mTCQhWUJrIcsC+yQMGEYmPUWxSzMDkUWdmtga2qLYWqmpoCCIpsK2ZBZjmECCxnWGcgBm2FJnDjBceKD+Iht2VZs+ZBkWZal7v7uH/1IkRXZaqsfqdU/fV5VXc/RPz3P96k4n+d5fs/R5u6IiEhYEpUuQERE4qdwFxEJkMJdRCRACncRkQAp3EVEAqRwFxEJkMJdRCRACncRkQAp3EVEApSq1IpbWlp8xYoVlVq9iEhVeu655467e+tE7SoW7itWrGDLli2VWr2ISFUyswOltFO3jIhIgBTuIiIBUriLiARI4S4iEiCFu4hIgBTuIiIBUriLiARI4S4iEiCFu4hIgCr2hOpM8t3NB8edf+f6ZdNciYhIPHTkLiISIIW7iEiAFO4iIgFSuIuIBEjhLiISIIW7iEiAFO4iIgFSuIuIBEjhLiISIIW7iEiAFO4iIgFSuIuIBEjhLiISIIW7iEiAFO4iIgFSuIuIBEjhLiISIIW7iEiAFO4iIgGaMNzNbKmZ/cLMdpjZdjP79DhtzMy+ZmZ7zGybmd0wNeWKiEgpSvmB7BzwGXd/3swagefM7FF33zGqzbuB1dFnPfDNaCgiIhUw4ZG7u3e6+/PR+GlgJ7BkTLPbgQe96Gmg2cwWxV6tiIiU5JL63M1sBXA9sHnMV0uAjlHTh3j9DkBERKZJyeFuZg3AD4E/dffeyazMzDaa2RYz29LV1TWZRYiISAlKCnczS1MM9u+4+4/GaXIYWDpqui2adx53v9fd17n7utbW1snUKyIiJSjlbhkDvgXsdPe/vUCzh4GPRHfN3AT0uHtnjHWKiMglKOVumVuAu4AXzWxrNO+/AMsA3P0eYBPwHmAP0A/8UfyliohIqSYMd3f/FWATtHHgk3EVJSIi5dETqiIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAZow3M3sfjM7ZmYvXeD7W82sx8y2Rp8vxF+miIhcilQJbR4Avg48eJE2/+zuG2KpSEREyjbhkbu7PwmcnIZaREQkJnH1ud9sZi+Y2SNmdnVMyxQRkUkqpVtmIs8Dy929z8zeA/wYWD1eQzPbCGwEWLZsWQyrFhGR8ZR95O7uve7eF41vAtJm1nKBtve6+zp3X9fa2lruqkVE5ALKDnczW2hmFo3fGC3zRLnLFRGRyZuwW8bMvgfcCrSY2SHgi0AawN3vAT4AfMLMcsBZ4EPu7lNWsYiITGjCcHf3Oyb4/usUb5UUEZEZQk+oiogESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gEaMJwN7P7zeyYmb10ge/NzL5mZnvMbJuZ3RB/mSIicilKOXJ/ALjtIt+/G1gdfTYC3yy/LBERKceE4e7uTwInL9LkduBBL3oaaDazRXEVKCIily6OPvclQMeo6UPRvNcxs41mtsXMtnR1dcWwahERGc+0XlB193vdfZ27r2ttbZ3OVYuIzCpxhPthYOmo6bZonoiIVEgc4f4w8JHorpmbgB5374xhuSIiMkmpiRqY2feAW4EWMzsEfBFIA7j7PcAm4D3AHqAf+KOpKlZEREozYbi7+x0TfO/AJ2OrSEREyqYnVEVEAqRwFxEJkMJdRCRACncRkQAp3EVEAqRwFxEJkMJdRCRACncRkQAp3EVEAqRwFxEJkMJdRCRACncRkQAp3EVEAqRwjxztHaC7f7DSZYiIxELhDnSc7Ocbv9jDg08doPgGYxGR6jbrw/3QqX7+7ukDALzaO8D+E/0VrkhEpHyzOtx7B4b42APPkisU+A9vW0VtOsmv9x6vdFkiImWb1eH+2X94gX1dZ/jw+uUsaa7lzSvmsrOzV33vIlL1Zm24nx3M89jOY3zsLe2sam0AYP3K+bjD5ldOVrg6EZHyzNpwf+lID/mCc+OKeSPz5tZluGpRE8/uP8lQvlDB6kREyjNrw33rwW4ArlvWfN78m1fNp38wz7ZD3ZUoS0QkFrM33Du6aZtbS0tDzXnzV7bUs6Cxhi37T1WoMhGR8s3qcL9uafPr5psZVy1qouNUP/2DuQpUJiJSvlkZ7sdOD3C4++y44Q7Q3lJPweH5A+qaEZHqNCvDfbi//fpl44f7snl1GPDMKyemsSoRkfjMznDv6CaVMK5ePGfc77PpJIuba3lat0SKSJWateF+1aImsunkBdu0t9SztaObgaH8NFYmIhKPWRfu+YKz7VDPBfvbh7W31DOYK7C1Q/3uIlJ9Zl247+3qo+9cjmsnCPcV8+sxg8371DUjItVn1oX7yMNLE4R7bSbJlQubeGa/LqqKSPWZfeF+qJvGbIqVLfUTtl3fPo/nDpxiMKdXEYhIdZl94X6w+PBSImETtl3fPo+BoQIvHla/u4hUl1kV7oO5Ar89epo3Lhn/FsixbmwvvlTsafW7i0iVSVW6gOm0/8QZ8gXnDQsbS2r/s+1HWdBYw49/c5i5dRkA7ly/bCpLFBGJxaw6ct9zrA9g5P3tpWhvqefAiX7yBf22qohUj1kV7ruP9mF2aeG+srWBwXyBw6f026oiUj1KCnczu83Mfmtme8zsc+N8/1Ez6zKzrdHnj+MvtXx7uvpom1tLbebCT6aO1R7dVbPv+JmpKktEJHYThruZJYFvAO8G1gB3mNmacZr+vbtfF33ui7nOWOw+eprVC0rrbx/WUJNiYVOWvV19U1SViEj8SjlyvxHY4+773H0Q+D5w+9SWFb98wdl3/AyXLyi9S2ZYe2ux3z2nn94TkSpRSrgvATpGTR+K5o31fjPbZmYPmdnSWKqLUcfJfgZzhUmF+6qWBnIFp+PU2SmoTEQkfnFdUP1HYIW7rwUeBb49XiMz22hmW8xsS1dXV0yrLs3u6E6Z1ZM5cm+px4B96poRkSpRSrgfBkYfibdF80a4+wl3PxdN3ge8abwFufu97r7O3de1trZOpt5JG7kNchLhXptJsqg5y94uXVQVkepQSrg/C6w2s3YzywAfAh4e3cDMFo2afC+wM74S47H72GkWNmVpyqYn9fcrWxroONWv97uLSFWYMNzdPQf8CfAziqH9A3ffbmZ/YWbvjZp9ysy2m9kLwKeAj05VwZO191jfpPrbh61qrSdfcJ47cCrGqkREpkZJrx9w903ApjHzvjBq/PPA5+MtLT7uzp5jfXxw3eSv866YX0/C4Nd7j3PL5S0xViciEr9Z8YRqZ88AZwbzZR2516STLGmu5am9er+7iMx8syLcy7lTZrTLFzSytaObrtPnJm4sIlJBsyLch++UKefIHeCNbXMoODzyUmccZYmITJlZEu6nmVefYX5DTVnLWdiU5YrLGvjpCwp3EZnZZkm493H5JbwJ8mI2rF3MM/tP0tmjp1VFZOYKPtzdnd3H+rj8srjCvXhL///ZpqN3EZm5gg/3rtPn6O4fKvti6rCVrQ1cvbiJnyrcRWQGCz7ct3f2ArBmUVNsy9ywdjFbO7rpOKkf8BCRmSn4cN9xpBjuVy2OM9yLXTM6eheRmSr4cN/Z2cvSebWTfqfMeJbOq+O6pc384wtHYlumiEicgg/3HZ29sXbJDHv/DUvY0dmrJ1ZFZEYKOtz7B3O8cvwMaxbNiX3ZH1y3lAWNNXz18ZdjX7aISLmCDvddr57GHdbE2N8+LJtO8olbV/H0vpM8vU9H7yIyswQd7iMXUxdd2o9il+qOG5fR2ljDVx/bPSXLFxGZrLDDvbOXpmyKJc21U7L8bDrJJ962iqf2nWCzjt5FZAYJO9yP9LJmcRNmNmXruHN98ej9K4/txt2nbD0iIpci2HDPF5xdr/ZOycXU0bLpJOvb5/HUvhN89qFtfHfzQb67+eCUrlNEZCLBhvsrx88wMFSYkoupY920cj4r5tfz8AtHONGnd72LSOUFG+47puC1AxeSMOPfrWsjacbfb+kgVyhM+TpFRC4m3HA/0ks6aWX/QEepmusyvO/6JRw6dZbHdx6blnWKiFxIuOHe2cvqBY1kUtO3idcsmcObV8zlyZe7+MGWjmlbr4jIWOGGe3SnzHTbsHYxqxY08J9/uI0fPKuAF5HKCDLcX+0Z4HjfOa6ahv72sdLJBHfdtJzfXd3K3T/cxvef0Z0zIjL9ggz3//tysc/7lsvnV2T96WSCe+96E2+7opXP/ehFvrRpJ4M5XWQVkekTZLg/vvMYi+dkecNlU/PagVJk00n+x11v4s71y7j3yX184J5f88rxMxWrR0Rml+DC/Vwuz6/2HOdfXLlgSp9MLUU2neRL73sj9/zBDRw40c+Gr/0z3/zlXgaG8hWtS0TCl6p0AXHbvO8k/YN53nHVgkqXMuK2axaxtq2ZL/zkJf7qn3bx4FP7+bN3XsH7rl9CKpkY94nWO9cvm/5CRSQYwYX7E7uOUZNKcPPKlorWMV5gv/3Ky/jj313JX27ayWcf2sZXHtvNR25eTiqRoDaTrECVIhKqoMLd3Xli1zFuubxlxoblTSvn8+NP3sLPdxzl/l+9wl8+sot00njjkjlct3QuK1vrSVS4O0lEql9Q4b636wwHT/bz79+6stKlXJSZ8a6rF/Kuqxey/UgPX/zJdl483MPzB7uZU5vm6sVNrJhfx5vb55FOBndZRESmQVDh/otdxVsg337lzOlvH2u87pp/e0Mbv3ftYnZ29rK1o5tnXjnJnXs305RN8ZbVLdy8qoXfWTWflS31Fb9ILCLVIahwf3zXUa5c2DhlP84xldLJBGvbmlnb1sxgrsCi5iyP7TjK/9tznE0vvgrAvPoM17bNYW1bM1cvbuKKyxpZOq+OZEKBLyLnCybce/qH2LL/FBtneJdMKTKpBCf6Brl+2VyuW9rMyTOD7Iu6nI50D/DLl3cz/Lsg2XSC9pYGls+rY3lLHUvn1rFkbi1tzbUsbq6lviaY/8QicgmC+T//a0/sJu/OhrWLK11KrMyM+Q01zG+o4c3t87hz/TLOnMvx8tHT0aePX+0+zpYDp3h051HyhfN/DSqbTjCnNh19Mrx1dQuLmmtZPCfLwjlZFs2pnbEXn0Vk8oII9x1Henng1/u548ZlFXlZ2HQa22e/qrWBVa3F1xoX3Ok9O0TP2SG6+4foPjtEz9lBevqL8w6fOsuz+0++bplN2RSXNRXDvrWhhtbG4mdefYa59Rnm12dors0wpy5NY02KhLqBRGa8qg/3QsH585+8xJzaNHe/6w2VLqeiEmY012Vorsuw/AKv1RnKF4o7gIEhevqHovEcvWeHeOX4GV483EPfQI5cYfzfg00YNNSkaKpN05hN01CTpL4mRX0mRU06QTadJJtKkkklyCSNVDJBKmmkEwmSCSOdNNLJBKlkgkwqQU30yaaT1KaT1GaKw2w6STadoDadJKU7hkQuWdWH+0PPH+K5A6f46w+spbkuU+lyZrx0MjHSzXMh7s7AUIH+wRxnzuU4M5infzDP2aE8ZwdzDAwVGBjKMzCU50TfIJ09A5wbKjBUKDCUd3L5Ak5xRxLHb4YnE0ZN6rWdQXGYHNkp1KQS1GWS1GZS1GeSNNSkaMimaMqmaapN05RNFbul6tIjXVS16aTuPJKglRTuZnYb8FUgCdzn7l8e830N8CDwJuAE8Pvuvj/eUl9vX1cfX35kF+uWz+X9N7RN9epmDTMrHkFnkhfdCUzE3Sl4sbsoX/CRYXEccvkCuUJxZzBUcIbyxZ3DUC7aUeQKDOadXKFA7ryhj/xdT/9Q9HcFzuUKDOaiYf7ib+FMJ20k/BuzKRpqUjRmU9TXpKjLJKnPpKKzh+LOoyadIJ1MkEkmojMPIxMNU4nhoY18l0okSCeNZKI4nkzYyCc1ZqidjEyFCcPdzJLAN4B3AoeAZ83sYXffMarZx4FT7n65mX0I+Cvg96eiYCi+HOyeX+7jG7/cQ00qwX9/3zXqB56BzIykQRIjPc3XbPMF51wuz8BQITrjKJ55DAzm6Y/OOs5Gw4GhPD39Q5zLFRjI5RmMdhIX6pqK23BX1WvdVFGXVCZFXToZnZUUh3VR91ft6B3PcDdYqrjzGT67yYzq8sqmkmQzxb+ry6R0++wsUMqR+43AHnffB2Bm3wduB0aH++3Af4vGHwK+bmbmHsdJ+fl+c/AUn/mHF9jXdYbfu3Yxf77hKhY0ZuNejVS5ZMKoy6Qop6du+ExjKF88Yxg+68i5Uyi8Np334llJcR7F6ehMpbgMRsYLHp3RjPwdry03OhvJRevsP5eju3+QoXxhZIczmH/t7KUcmWRi5PpGbaa4k6jNFHcCNenhHUSSTDJBJmUjZyzp1PAZTHGHVPwYyVFnLyNnKWYkRobFnX3CjIQRDaPxxNh5hg1PJ85vm0wUx5MjZz2QjKaH1zXcZniZZszKs6NSwn0JMPr34g4B6y/Uxt1zZtYDzAeOx1HkaGZGoeB8+2M38rYrWuNevMiIhBmJKMRmmoIXdwD5Md1UudE7iqgba3jnNBh1Xw3mX+vyGt5x5PIFTg/kOJkfLP599F3efWQd+VHdatXKDGxk3LAx342Mc97EeaPD7c7/6wstZ3wff0s7f/avpvYGkGm9oGpmG4GN0WSfmf12ssu69e6R0RamYCdyIR+e2sVP67ZMMW3LzKRtmQE+E31GuZRtWV5Ko1LC/TCwdNR0WzRvvDaHzCwFzKF4YfU87n4vcG8phZXKzLa4+7o4l1kp2paZSdsyM2lbLq6U881ngdVm1m5mGeBDwMNj2jwM/GE0/gHgianobxcRkdJMeOQe9aH/CfAzirdC3u/u283sL4At7v4w8C3g78xsD3CS4g5AREQqpKQ+d3ffBGwaM+8Lo8YHgA/GW1rJYu3mqTBty8ykbZmZtC0XYeo9EREJz8y7x0tERMpWteFuZreZ2W/NbI+Zfa7S9UyWmS01s1+Y2Q4z225mn650TeUys6SZ/cbMflrpWsplZs1m9pCZ7TKznWZ2c6Vrmiwz+0/Rv7GXzOx7ZlY1T/+Z2f1mdszMXho1b56ZPWpmu6Ph3ErWWKoLbMtfR//GtpnZ/zaz5nLXU5XhPuqVCO8G1gB3mNmaylY1aTngM+6+BrgJ+GQVb8uwTwM7K11ETL4K/JO7XwlcS5Vul5ktAT4FrHP3ayjeHFFNNz48ANw2Zt7ngMfdfTXweDRdDR7g9dvyKHCNu68FXgY+X+5KqjLcGfVKBHcfBIZfiVB13L3T3Z+Pxk9TDI8lla1q8sysDfjXwH2VrqVcZjYHeCvFu8Fw90F3765sVWVJAbXRsyh1wJEK11Myd3+S4p14o90OfDsa/zbwb6a1qEkab1vc/efunosmn6b4PFFZqjXcx3slQtUG4jAzWwFcD2yubCVl+QpwN3Dx1zJWh3agC/ifUTfTfWZWX+miJsPdDwN/AxwEOoEed/95Zasq22Xu3hmNvwpcVsliYvQx4JFyF1Kt4R4cM2sAfgj8qbv3VrqeyTCzDcAxd3+u0rXEJAXcAHzT3a8HzlA9p/7nifqjb6e4w1oM1JvZH1S2qvhED01W/a1/ZvZfKXbVfqfcZVVruJfySoSqYWZpisH+HXf/UaXrKcMtwHvNbD/FrrK3m9n/qmxJZTkEHHL34TOphyiGfTX6l8Ar7t7l7kPAj4DfqXBN5TpqZosAouGxCtdTFjP7KLAB+HAcT/hXa7iX8kqEqmDFd5F+C9jp7n9b6XrK4e6fd/c2d19B8b/JE+5etUeH7v4q0GFmw6/vewfnv+q6mhwEbjKzuujf3Duo0ovDo4x+7ckfAj+pYC1liX4Q6W7gve7eH8cyqzLcowsPw69E2An8wN23V7aqSbsFuIviUe7W6POeShclI/4j8B0z2wZcB3ypwvVMSnT28RDwPPAixf/3q+YJTzP7HvAU8AYzO2RmHwe+DLzTzHZTPDP58sWWMVNcYFu+DjQCj0YZcE/Z69ETqiIi4anKI3cREbk4hbuISIAU7iIiAVK4i4gESOEuIhIghbuISIAU7iIiAVK4i4gE6P8DBjGyVYBAU0MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(np.log1p(list(word_count.values())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f932f63a048>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAD8CAYAAABZ/vJZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X2QXNV95vHv093zopfRoHeEXhg5iNjCL2BUsp1kXZtgY3B5EUlwLEg5eJcsroop29lkN1ApO142rjWpJDgOrDfEkBAqWNjEiScOMWtekrUrsczgsBgJCw1YQhJCGo2EpJE0krr7t3/0balpejQ9PT1vfZ9PVZe6zz339L3dKj0695x7WhGBmZnZWGWm+gDMzGxmcoCYmVlDHCBmZtYQB4iZmTXEAWJmZg1xgJiZWUMcIGZm1hAHiJmZNcQBYmZmDclN9QE0w6JFi6Knp2eqD8PMbEZ5+umnD0TE4kb3b4kA6enpoa+vb6oPw8xsRpG0czz7+xKWmZk1xAFiZmYNcYCYmVlDHCBmZtYQB4iZmTXEAWJmZg1xgJiZWUMcIGZm1hAHiJmZNaQl7kRvlgc3v1yz/IZ3rZrkIzEzm/7cAzEzs4Y4QMzMrCF1BYikqyRtk9Qv6dYa2zskPZRs3yypJylfKOlJSUOS7qqo3yXpmYrHAUlfTLZ9TNJAxbZfb86pmplZM406BiIpC9wNvB/YDTwlqTcitlZUuwk4FBEXSdoI3AF8BBgGPgO8NXkAEBFHgUsr3uNp4BsV7T0UEbc0fFZmZjbh6umBrAf6I+KliDgFbAI2VNXZANyfPH8YuEKSIuJYRHyPUpDUJOliYAnw3TEfvZmZTZl6AmQ5sKvi9e6krGadiMgDh4GFdR7DRko9jqgo+2VJz0p6WNLKOtsxM7NJNB0G0TcCX614/fdAT0S8HfgOZ3s2ryPpZkl9kvoGBgYm4TDNzKxSPQGyB6jsBaxIymrWkZQDuoHB0RqW9A4gFxFPl8siYjAiTiYvvwJcXmvfiLgnItZFxLrFixv+RUYzM2tQPQHyFLBG0mpJ7ZR6DL1VdXqBG5Pn1wFPVF2SGsn1vL73gaRlFS+vAZ6vox0zM5tko87Cioi8pFuAR4EscF9EbJF0O9AXEb3AvcADkvqBg5RCBgBJO4B5QLuka4ErK2Zw/Qrwwaq3/KSka4B80tbHxnF+Y1KMICNN1tuZmc1oqq+jML2tW7cu+vr6xtXGd7cP8Ov39/FfP/DTzG5/fa56KRMza0WSno6IdY3uPx0G0aeFXQdPcDJfZHDo1FQfipnZjOAASRSKRQAOnzg9xUdiZjYzOEAS+WLpUt6RYQeImVk9HCCJQjlA3AMxM6uLAyRR7oH4EpaZWX0cIIkzPZDh/BQfiZnZzOAASeQL7oGYmY2FAySRT2ZhHTlxmla4N8bMbKI5QBLlMZB8MThxqjDFR2NmNv05QBLlMRCAw57Ka2Y2KgdIojwGAp7Ka2ZWDwdIonwnOsDhE56JZWY2GgdIIl8MOtsyCN+NbmZWj1GXc0+LQjFoy2Zoy2Y8ldfMrA4OkES+WPotkK7OnMdAzMzq4EtYiUIxyAjmdba5B2JmVgcHSCJfDLIZMW9W2+vGQIZO5vn4A30cGDp5jr3NzNLHAZIoFItkJLpntTF8usipfGlW1nN7DvPoln307Tg0xUdoZja9OEAS+UJpDGReZ2lYqDwOsn3/EIB7IGZmVRwgiUIxyGSge1YbULobvVAMXhpwgJiZ1VJXgEi6StI2Sf2Sbq2xvUPSQ8n2zZJ6kvKFkp6UNCTprqp9/ilp85nkseRcbU208iyseUmAHDlxml0Hj3MyuZTlADEze71RA0RSFrgbuBpYC1wvaW1VtZuAQxFxEXAncEdSPgx8BvjtEZr/1Yi4NHnsH6WtCVUoB0hn0gM5cZrt+4cQcP68Tg4cPTUZh2FmNmPU0wNZD/RHxEsRcQrYBGyoqrMBuD95/jBwhSRFxLGI+B6lIKlXzbbGsH9DThdKg+jtuQyz2rIcPnGa/v1HWblgNqsXzXEPxMysSj0BshzYVfF6d1JWs05E5IHDwMI62v6L5PLVZypCotG2xqU8BgKlcZB9R4bZfegEFy2Zy6KuDgeImVmVqRxE/9WIeBvw75LHR8eys6SbJfVJ6hsYGBj3weSLQTbJsHmzcuwYPE4Aa5bMZdHcdg4M+RKWmVmlepYy2QOsrHi9IimrVWe3pBzQDQyeq9GI2JP8eVTSg5Qulf1VvW1FxD3APQDr1q0b908IlsdAgDPjIB25DCvmz+a72wcYOpnn/n/ZQVv2bObe8K5V431bM7MZq54eyFPAGkmrJbUDG4Heqjq9wI3J8+uAJ+IcvwsrKSdpUfK8DfgQ8FwjbTVLPlnKBDgzE+unFs8lmxFzO0o5OzTsZd7NzMpG7YFERF7SLcCjQBa4LyK2SLod6IuIXuBe4AFJ/cBBSiEDgKQdwDygXdK1wJXATuDRJDyywGPAnye7jNjWRCoUi2d6F+V7QdYsnQvA3OTmwqGTeebPaZ+MwzEzm/bqWo03Ih4BHqkq+2zF82HgwyPs2zNCs5ePUH/EtiZSvhh05EpdkFULZrOkq4O3nD8P4GwP5KR7IGZmZV7OPVGouIS1dF4nn37fxWe2+RKWmdkbeSmTRHktrFrKAXLUPRAzszMcIIlCspx7Lblshs62jC9hmZlVcIAk8sWReyBQ6oU4QMzMznKAJArF4pk70WuZ25HzGIiZWQUHSMI9EDOzsXGAJAqjBUhnjqGT/q10M7MyB0iinh7I8Oki+UJxEo/KzGz6coAk8oXRxkBKd6f7MpaZWYkDBCgWg2Iwag8EHCBmZmUOEKCQrNU42hgIOEDMzMocIJQG0AGy5/jdQy9nYmb2eg4QSgPoAJkR7kQHX8IyM6vmAAEKhdEvYbXnMrTnvJyJmVmZAwTIF0tTc8/RAQF8M6GZWSUHCGfHQM51CQscIGZmlRwgnB0DyZ7jEhZ4PSwzs0oOECp6IKMFSKd7IGZmZQ4QKmdhnbve3I4cJ04VzgSOmVma1RUgkq6StE1Sv6Rba2zvkPRQsn2zpJ6kfKGkJyUNSbqrov5sSf8g6ceStkj6QsW2j0kakPRM8vj18Z/muRXODKKPfgkrgGOn3AsxMxs1QCRlgbuBq4G1wPWS1lZVuwk4FBEXAXcCdyTlw8BngN+u0fQfRsSbgcuAn5V0dcW2hyLi0uTxlTGdUQPy9V7C8s2EZmZn1NMDWQ/0R8RLEXEK2ARsqKqzAbg/ef4wcIUkRcSxiPgepSA5IyKOR8STyfNTwA+BFeM4j3HJ13EfCECXlzMxMzujngBZDuyqeL07KatZJyLywGFgYT0HIOk84D8Aj1cU/7KkZyU9LGllPe2MR2EMYyDgHoiZGUzxILqkHPBV4EsR8VJS/PdAT0S8HfgOZ3s21fveLKlPUt/AwMC4jiNf7xiIeyBmZmfUEyB7gMpewIqkrGadJBS6gcE62r4H2B4RXywXRMRgRJxMXn4FuLzWjhFxT0Ssi4h1ixcvruOtRlbvJayOXJb2rJczMTOD+gLkKWCNpNWS2oGNQG9VnV7gxuT5dcATEXHOua6Sfp9S0Hy6qnxZxctrgOfrOMZxqfcSFpR6IUeH/dO2Zma50SpERF7SLcCjQBa4LyK2SLod6IuIXuBe4AFJ/cBBSiEDgKQdwDygXdK1wJXAEeB3gR8DP1Tpf/53JTOuPinpGiCftPWxJp3riOq9Ex28nImZWdmoAQIQEY8Aj1SVfbbi+TDw4RH27Rmh2Zr/WkfEbcBt9RxXs9R7JzqUAuTA0MlR65mZtTrfiU7994FAaSqveyBmZg4QoOJO9HrGQDpyHPdyJmZmDhAYWw/EU3nNzEocIFT+Jnodl7D807ZmZoADBKi4D2S0nyQE5na2ATDkqbxmlnIOECpnYY1ed657IGZmgAMEGOMYSBIgR70elpmlnAOEyllYowdIey5DR87LmZiZOUCo7IHUV39uR849EDNLPQcIY7sTHfzb6GZm4AAB4HSdq/GWdXXk/JsgZpZ6DhDGdic6uAdiZgYOEGBss7AA5na0ceJ0gZP5wkQelpnZtOYAoTQGIo3tEhbA4NCpiTwsM7NpzQFCqQeSq3cKFmfXwxo46mXdzSy9HCCUeiDZsQRIhwPEzMwBQmktrFy9I+iUfhME8A9LmVmqOUAozcLKZevvgcxxD8TMzAECYx8Dactm6GzLuAdiZqnmAGHsYyBQmso74AAxsxSrK0AkXSVpm6R+SbfW2N4h6aFk+2ZJPUn5QklPShqSdFfVPpdL+lGyz5ek0hxaSQskfUfS9uTP+eM/zXMr9UDGlqVdnTkOHPU0XjNLr1H/1ZSUBe4GrgbWAtdLWltV7SbgUERcBNwJ3JGUDwOfAX67RtNfBv4zsCZ5XJWU3wo8HhFrgMeT1xOqsR5Izj0QM0u1ev7bvR7oj4iXIuIUsAnYUFVnA3B/8vxh4ApJiohjEfE9SkFyhqRlwLyI+H5EBPBXwLU12rq/onzCjHUMBEr3ghzwILqZpVg9AbIc2FXxendSVrNOROSBw8DCUdrcPUKbSyNib/L8VWBpHcc4LoViccw9kK6OHEdP5hk+7eVMzCydpvUgetI7iVrbJN0sqU9S38DAwLjeJ19o7BIWeCqvmaVXPQGyB1hZ8XpFUlazjqQc0A0MjtLmihHa3Jdc4ipf6tpfq4GIuCci1kXEusWLF9dxGiPLF2NM94FAxXImHgcxs5SqJ0CeAtZIWi2pHdgI9FbV6QVuTJ5fBzyR9B5qSi5RHZH07mT21a8B36zR1o0V5RMmXwyyY52F1dEG4HEQM0ut3GgVIiIv6RbgUSAL3BcRWyTdDvRFRC9wL/CApH7gIKWQAUDSDmAe0C7pWuDKiNgK/Abwl8As4B+TB8AXgK9JugnYCfxKM070XArFYkOD6AD7HCBmllKjBghARDwCPFJV9tmK58PAh0fYt2eE8j7grTXKB4Er6jmuZmlkDKSrM0d7LsPug8cn6KjMzKa3aT2IPlkKDUzjzUisWjCbHYPHJuiozMymNwcI5TGQsQUIQM/C2ewcdA/EzNLJAUKpB9KWHftHceHCOewYPEaxOOJ8ATOzluUAYXw9kOHTRfZ7IN3MUsgBQmOzsAB6Fs0B8DiImaWSA4Tx9EBKAbLTAWJmKeQAobFZWADLujtpy4odHkg3sxRygFC+D2TsH0Uum2Hl/NnugZhZKjlAaLwHAnDhwtn85IB7IGaWPg4QkjGQMS6mWNazaA47B49xjqW/zMxakgOExmdhQWkg/fipglflNbPUcYDQ2FpYZRcunA3gO9LNLHUcIDT2k7Zl5am8Ow54IN3M0qWu1XhbXaGB3wMBeHDzyxSKQUbwD8/u5XShNA5yw7tWNfsQzcymHfdAgPw4xkCyGTF/djuDx041+ajMzKa31AdIsRgUg4bHQAAWzm1n8JgH0c0sXVIfIIVk+m2jPRCABXM6GBw65am8ZpYqDpBkKfZcA8u5ly2c087JfJFjpwrNOiwzs2kv9QGSL46/B7JwbjsAB30viJmlSOoDpJDMnBrPGMiiOR0AHPBAupmlSF0BIukqSdsk9Uu6tcb2DkkPJds3S+qp2HZbUr5N0geSsp+W9EzF44ikTyfbPidpT8W2DzbnVGvLF4sA5BpcygTgvDltZAQH/MNSZpYio94HIikL3A28H9gNPCWpNyK2VlS7CTgUERdJ2gjcAXxE0lpgI3AJcAHwmKSLI2IbcGlF+3uAv61o786I+MPxn97oymMg4+mB5DIZFnd1sPfwcLMOy8xs2qunB7Ie6I+IlyLiFLAJ2FBVZwNwf/L8YeAKSUrKN0XEyYj4CdCftFfpCuDFiNjZ6EmMRzPGQACWdc9i7+ETzTgkM7MZoZ4AWQ7sqni9OymrWSci8sBhYGGd+24EvlpVdoukZyXdJ2l+rYOSdLOkPkl9AwMDdZxGbWd7IOMbDrqgu5Mjw3mGTubH1Y6Z2UwxpYPoktqBa4CvVxR/GfgpSpe49gJ/VGvfiLgnItZFxLrFixc3fAxN64GcNwuAV30Zy8xSop4A2QOsrHi9IimrWUdSDugGBuvY92rghxGxr1wQEfsiohARReDPeeMlr6YqJIPo4xkDAVg2rxOAV17zZSwzS4d6AuQpYI2k1UmPYSPQW1WnF7gxeX4d8ESUbsvuBTYms7RWA2uAH1Tsdz1Vl68kLat4+YvAc/WeTCPKCyCOtwcyuyNH96w2j4OYWWqMOgsrIvKSbgEeBbLAfRGxRdLtQF9E9AL3Ag9I6gcOUgoZknpfA7YCeeATEVEAkDSH0syuj1e95R9IuhQIYEeN7U3VjFlYZcu6Oz0Ty8xSo67l3CPiEeCRqrLPVjwfBj48wr6fBz5fo/wYpYH26vKP1nNMzXJmDGQc94GULeuexbZXjzJ8ukBnW3bc7ZmZTWe+E/3MGMj4P4pl3Z0EsO3Vo+Nuy8xsukt9gOSbNAYCcEEyE2vr3iPjbsvMbLpLfYAUmjSNF+C82W105DJsfcUBYmatL/UB0swxkIzEsu5O90DMLBVSHyDNuhO9bNl5s3h+7xGKRf+4lJm1ttQHSLPuRC+7oLuT46cK7Dx4vCntmZlNV6kPkGbdiV62rLs0kL7llcNNac/MbLpKfYA0uweypKuDXEYeSDezlpf6AGnmnehQ+m31Ny2ewwv7fC+ImbW21AfI2ftAmvdRrFnaxQv7hprWnpnZdJT6ADnTA2nCNN6yi5d0sevQcU6cKjStTTOz6Sb1AdLsMRCAi5fOJQL697sXYmatywHS5FlYULqEBbDN4yBm1sIcIE1cC6usZ+Fs2rMZtjtAzKyFpT5Amj0LCzwTy8zSIfUBcnYMpLkfxcWeiWVmLS71AdLsO9HLLl46lz2vneDYyXxT2zUzmy5SHyATMQsLzg6kb/dMLDNrUakPkEIxyAgyTe+BlALkBf86oZm1qLoCRNJVkrZJ6pd0a43tHZIeSrZvltRTse22pHybpA9UlO+Q9CNJz0jqqyhfIOk7krYnf84f3ymeW74YTR//AFi1YDYduYwH0s2sZeVGqyApC9wNvB/YDTwlqTcitlZUuwk4FBEXSdoI3AF8RNJaYCNwCXAB8JikiyOifIv2z0fEgaq3vBV4PCK+kITVrcDvjOMcz6lQjKaPfzy4+WUAFsxp559fGOBNyesb3rWqqe9jZjaV6vmv93qgPyJeiohTwCZgQ1WdDcD9yfOHgSskKSnfFBEnI+InQH/S3rlUtnU/cG0dx9iwfCGaPv5RtnReJ/uPnpyQts3Mplo9AbIc2FXxendSVrNOROSBw8DCUfYN4P9IelrSzRV1lkbE3uT5q8DSOo6xYYVisanrYFVa2tXB4ROnGT7tNbHMrPVM5SD6z0XEO4GrgU9Iem91hYgISkHzBpJultQnqW9gYKDhgyiNgUxMgCyZ1wnA/iPDE9K+mdlUqidA9gArK16vSMpq1pGUA7qBwXPtGxHlP/cDf8vZS1v7JC1L2loG7K91UBFxT0Ssi4h1ixcvruM0apuIMZCypUmA7Dviy1hm1nrqCZCngDWSVktqpzQo3ltVpxe4MXl+HfBE0nvoBTYms7RWA2uAH0iaI6kLQNIc4ErguRpt3Qh8s7FTq89EzcICOG92G21Zse+oeyBm1npGnYUVEXlJtwCPAlngvojYIul2oC8ieoF7gQck9QMHKYUMSb2vAVuBPPCJiChIWgr8bWmcnRzwYER8O3nLLwBfk3QTsBP4lSae7xtMZA8kI7Gkq5P97oGYWQsaNUAAIuIR4JGqss9WPB8GPjzCvp8HPl9V9hLwjhHqDwJX1HNczXC6UJywMRCApfM6fDe6mbUk34k+gT0QKI2DHB3Oc/yU18Qys9aS+gDJT3CALOnyQLqZtabUB0ihGOQm6D4QKF3CAtjnqbxm1mJSHyATOQsLoHtWGx25DPs9E8vMWkzqA6RQnNhBdEksndfpS1hm1nJSHyD5wsSOgQAs6epg35FhSrfGmJm1htQHyESPgUBpJtbxUwUODJ2a0PcxM5tMqQ+Q0iysif0YykuabPdvg5hZC0l9gBQmcDHFsvJMrG0OEDNrIakPkIm+DwRgbkeOWW1ZXtjnO9LNrHWkPkAmehYWnJ2J5Z+3NbNWkvoAmYweCJQuY72w76hnYplZy0h9gEzGGAicXRPrVd+RbmYtIvUBUroPZOI/hiXJQLrHQcysVThAJmEMBGBpl6fymllrSX2AFIpBdoJvJASY05Fj0dwOfvyqA8TMWkPqAyQ/SWMgAJeu7Ob7Lw16IN3MWkLqA6RQmNjVeCtdufZ8dh86wZZXjkzK+5mZTaTUB0h+EtbCKnvf2qVkBI9ueXVS3s/MbCKlPkAm+idtKy2Y0867Vi/k2885QMxs5qsrQCRdJWmbpH5Jt9bY3iHpoWT7Zkk9FdtuS8q3SfpAUrZS0pOStkraIulTFfU/J2mPpGeSxwfHf5ojm6xZWGUfuGQp2/cP8eKAp/Oa2cw2aoBIygJ3A1cDa4HrJa2tqnYTcCgiLgLuBO5I9l0LbAQuAa4C/lfSXh74rYhYC7wb+ERVm3dGxKXJ45FxneE5FItBMZi0HgjAlZecD/gylpnNfLk66qwH+iPiJQBJm4ANwNaKOhuAzyXPHwbukqSkfFNEnAR+IqkfWB8R/wrsBYiIo5KeB5ZXtTnhCslsqMnqgTy4+WUAVsyfxYObX+a8We0A3PCuVZPy/mZmzVTPJazlwK6K17uTspp1IiIPHAYW1rNvcrnrMmBzRfEtkp6VdJ+k+bUOStLNkvok9Q0MDNRxGm9UKJYCZDLuRK90ybJ57D50gteO+wemzGzmmtJBdElzgb8BPh0R5bmtXwZ+CriUUi/lj2rtGxH3RMS6iFi3ePHiht4/X5zcHkjZJRd0A7B1r6fzmtnMVU+A7AFWVrxekZTVrCMpB3QDg+faV1IbpfD464j4RrlCROyLiEJEFIE/p3QJbUIUCuUeyOQGyKKuDpZ0dfCjPYcn9X3NzJqpngB5ClgjabWkdkqD4r1VdXqBG5Pn1wFPROl2615gYzJLazWwBvhBMj5yL/B8RPxxZUOSllW8/EXgubGeVL3yxSLApN0HUunSleexc/A4g0MnJ/29zcyaYdQAScY0bgEeBZ4HvhYRWyTdLumapNq9wMJkkPy/ALcm+24BvkZpcPzbwCciogD8LPBR4BdqTNf9A0k/kvQs8PPAbzbrZKudHQOZ/AC5bNV8BDz98qFJf28zs2aoZxYWyVTaR6rKPlvxfBj48Aj7fh74fFXZ94Ca/2pHxEfrOaZmmKoxEIDuWW2sWTqXH+48NKk3M5qZNUuq70TPF6ZmFlbZ5Rcu4Mhwnu9ub2wWmZnZVEp3gJTHQKbof/9vOb+L2e1Zvt63e0re38xsPFIdIFM5BgKQy2a4dOV5fGfrPg4d8z0hZjazpDpAymMgbVMwC6vs8gvnc6pQ5JvPVM+MNjOb3lIdIFN1J3qlZd2zeNvybh74/k5O5YtTdhxmZmOV6gCZyllYlT55xRpeHDjGl//pxSk9DjOzsUh1gBSSQfSpnkL7/rVLueYdF/CnT2zneS9vYmYzRKoDpDyNd6p7IA9ufpm3Le+moy3LTfc/xQP/uvPMyr1mZtNVqgNkqmdhVZrTkWPDOy7gldeGfV+Imc0IqQ6QM2MgUzgLq9Jbl3fztuXdPP78fnYdPD7Vh2Nmdk6pDpDpMAur2rWXLmferBwP/uBl3xtiZtPa9PmXcwpMl1lYlWa1Z7lh/YUMnczz6YeeoZgco5nZdJPqAJkus7CqLZ8/iw+9fRn//MIAf/ydF3x/iJlNS3WtxtuqpmMPpGx9zwIyEnc92c9f/esO3veWpVx72XLee3Fjv75oZtZsqQ6Q6TQLq5ok3rlqPnM7cmx55Qj/+NyrfOPf9nD9+lX8z19621QfnplZui9hnT5zH8j0/BiyGfGWZfO47vIV3Hb1m1k5fxbf+OFudhw4NtWHZmaW7gA5MwYyTabxnksum2Hj+lVkJH7jr3/I8OnCVB+SmaVcqgPkzGq80/ASVi3zZ7dz3eUr2Lr3CJ/5u+f4f7teY9urR3nltRNTfWhmlkIeA2F6joGM5C3L5vHx976JP/u/L/H1p8/+ENU7V53HjT/Tw9VvXUZ7LtX/LzCzSVLXvzSSrpK0TVK/pFtrbO+Q9FCyfbOknopttyXl2yR9YLQ2Ja1O2uhP2mwf3ymOLD/Nx0BGsmrBbD7+3jfxa+++kOvXr+KqS85n5+BxPrXpGX7mC4/zxcdeYHDo5Jn6B4ZO8s1n9vAPz+7l6Z0H2fPaCSJ8f4mZjc+oPRBJWeBu4P3AbuApSb0RsbWi2k3AoYi4SNJG4A7gI5LWAhuBS4ALgMckXZzsM1KbdwB3RsQmSf87afvLzTjZamd6IDNgDKSSJC5cOOd1ZT+3ZhEv7h/iX14c5IuPbeeuJ/p56/JuDh47xa5Dx6nOi5ULZvGLl63gly5bTs+i17dVLAa7Dh0nI7Fi/iykmfX5mNnkqOcS1nqgPyJeApC0CdgAVAbIBuBzyfOHgbtU+ldnA7ApIk4CP5HUn7RHrTYlPQ/8AnBDUuf+pN0JCZDOtgxLujqm5X0gY5WRWLO0izVLu9h/dJh/6R/kmV2vsairnV948xLevHQe2Yw4fOI0h46fYusrR/jTJ7bzpce3s6Srg0VzO1jU1cHhE6d54dWjnEgG6RfMaeftK7rpWTiHXEZks6Itk6GzLUNHLktne5Y57Vlmt+eY3Z6lLZuhPSfashmymbN/5gvB6UKRfDHIZURHLkN78mjLZmhLeoFBEAGFCPKFIF8skstk6Mhl6GzLksuKrEQmI4ZO5nn18DD7jgxTKAZL5nWwpKuT82a1kWmB79RsuqsnQJYDuype7wbeNVKdiMhLOgwsTMq/X7Xv8uR5rTYXAq9FRL5G/ab76Ht6+Oh7eiaq+SmzpKuTay9bzrWXvfGjO7+7E4B3v2khh0+c5tndrzFw9CRHh/O8uH+IjrYMl606j/PndVKIYPehE/x471G+/9IgxSj1TgrFYLpfAJMoBY1E9dGKUriUw6rWvmfqnXl+tryyDQkieMN7VNapbheo+b7VdcpPo0b90TqFjs/2aKTLAAAEzklEQVTWUv76a/09+OyH1rJx/apJPyaYwYPokm4Gbk5eDkna1oRmFwEHmtDOTJTWc0/reYPPvSXO/fr/AdePbZfKc79wPO9dT4DsAVZWvF6RlNWqs1tSDugGBkfZt1b5IHCepFzSC6n1XgBExD3APXUcf90k9UXEuma2OVOk9dzTet7gc/e5j18904+eAtYks6PaKQ2K91bV6QVuTJ5fBzwRpWk+vcDGZJbWamAN8IOR2kz2eTJpg6TNbzZ+emZmNlFG7YEkYxq3AI8CWeC+iNgi6XagLyJ6gXuBB5JB8oOUAoGk3tcoDbjngU9ERAGgVpvJW/4OsEnS7wP/lrRtZmbTjHw/wFmSbk4ujaVOWs89recNPnefexPacoCYmVkjZtYt2GZmNm04QBh9qZaZTtJKSU9K2ippi6RPJeULJH1H0vbkz/lJuSR9Kfk8npX0zqk9g/GRlJX0b5K+lbyuuVzOuZbkmYkknSfpYUk/lvS8pPek6Dv/zeTv+nOSviqps1W/d0n3Sdov6bmKsjF/z5JuTOpvl3RjrfeqlvoA0dmlWq4G1gLXq7QESyvJA78VEWuBdwOfSM7xVuDxiFgDPJ68htJnsSZ53MwErQQwiT4FPF/xurxczkXAIUrL5UDFkjzAnUm9mexPgG9HxJuBd1D6DFr+O5e0HPgksC4i3kppok55iaVW/N7/EriqqmxM37OkBcDvUbqhez3we+XQOaeISPUDeA/waMXr24Dbpvq4Jvicv0lpHbJtwLKkbBmwLXn+Z8D1FfXP1JtpD0r3Ej1OaYmcb1G6SfsAkKv+/inNCnxP8jyX1NNUn0OD590N/KT6+FPynZdXxliQfI/fAj7Qyt870AM81+j3TOlexD+rKH9dvZEeqe+BUHuplglbPmWqJd3zy4DNwNKI2JtsehVYmjxvpc/ki8B/A4rJ63Mtl/O6JXmA8pI8M9FqYAD4i+Ty3VckzSEF33lE7AH+EHgZ2Evpe3yadHzvZWP9nhv6/h0gKSJpLvA3wKcj4kjltij9t6OlpuRJ+hCwPyKenupjmQI54J3AlyPiMuAYZy9jAK35nQMkl142UArRC4A5vPEST2pM5PfsAKlvqZYZT1IbpfD464j4RlK8T9KyZPsyYH9S3iqfyc8C10jaAWyidBnrT0iWy0nqVJ7bmfPW65fkmYl2A7sjYnPy+mFKgdLq3znA+4CfRMRARJwGvkHp70IavveysX7PDX3/DpD6lmqZ0SSJ0h39z0fEH1dsqlyCpnLZmF7g15IZG+8GDld0h2eMiLgtIlZERA+l7/WJiPhVRl4uZ6QleWaciHgV2CXpp5OiKyitCNHS33niZeDdkmYnf/fL597y33uFsX7PjwJXSpqf9OCuTMrObaoHf6bDA/gg8ALwIvC7U308E3B+P0epC/ss8Ezy+CCl67yPA9uBx4AFSX1Rmpn2IvAjSrNZpvw8xvkZ/HvgW8nzN1Fak60f+DrQkZR3Jq/7k+1vmurjHuc5Xwr0Jd/73wHz0/KdA/8d+DHwHPAA0NGq3zvwVUpjPacp9TxvauR7Bv5T8hn0A/+xnvf2nehmZtYQX8IyM7OGOEDMzKwhDhAzM2uIA8TMzBriADEzs4Y4QMzMrCEOEDMza4gDxMzMGvL/AQqzUs9ftlJ9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(list(map(len,corpora)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparing corpora for language modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "MIN_LENGTH = 2\n",
    "MAX_LENGTH = 180\n",
    "VOCAB_SIZE = 10000 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_words = set([word for word,_  in word_count.most_common(5000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_corpora = [[word for word in sent.split() if word in freq_words] for sent in corpora]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_corpora = list(filter(lambda sent: len(sent)> MIN_LENGTH, train_corpora))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model will predict probability of the next word in each support response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = [],[]\n",
    "for sent in train_corpora:\n",
    "    for i,word in enumerate(sent):\n",
    "        if i>0:\n",
    "            X.append(sent[:i])\n",
    "            y.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [' '.join(sent) for sent in X]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# Prep Word Indexer: assign each word a number\n",
    "indexer = Tokenizer(lower=False, filters='', num_words=VOCAB_SIZE)\n",
    "indexer.fit_on_texts(X) \n",
    "\n",
    "# Convert each word in sent to num and zero pad\n",
    "X_seqs = pad_sequences(sequences=indexer.texts_to_sequences(X), \n",
    "                           maxlen=MAX_LENGTH, padding='post', truncating='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save indexer for reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('keras_indexer.pkl','wb') as f:\n",
    "    pickle.dump(indexer, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_encoded =  list(map(lambda w:indexer.word_index[w] if w in indexer.word_index else 0,y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_generator(X,y, batch_size):\n",
    "    X_batch, y_batch = [],[]\n",
    "    for i, (data, target) in enumerate(zip(X,y)):\n",
    "        X_batch.append(data)\n",
    "        y_batch.append(target)\n",
    "        if (i+1) % batch_size ==0:\n",
    "            yield np.stack(X_batch), to_categorical(y_batch, VOCAB_SIZE)\n",
    "            X_batch, y_batch = [],[]\n",
    "    yield np.stack(X_batch), to_categorical(y_batch, VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 180, 32)           320000    \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 128)               82432     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10000)             1290000   \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 10000)             0         \n",
      "=================================================================\n",
      "Total params: 1,692,432\n",
      "Trainable params: 1,692,432\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# build the model: a single LSTM\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=VOCAB_SIZE, output_dim=32, input_length=MAX_LENGTH, mask_zero=True))\n",
    "model.add(LSTM(128, input_shape=(MAX_LENGTH, VOCAB_SIZE)))\n",
    "model.add(Dense(VOCAB_SIZE))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "optimizer = RMSprop(lr=0.01)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=optimizer)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=128\n",
    "model.fit_generator(batch_generator(X_seqs, y_encoded, batch_size), \n",
    "                    steps_per_epoch=int(np.ceil(len(X_seqs)/batch_size)), epochs=100, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('language_model.h5')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow_p36]",
   "language": "python",
   "name": "conda-env-tensorflow_p36-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}