First sentence baseline-ready.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/jovyan/work/_Headline_generation/News_Headline_Generation/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import rouge\n",
    "from tqdm import tqdm, trange\n",
    "from multiprocessing import Pool\n",
    "from joblib import Parallel, delayed\n",
    "import re\n",
    "from ipymarkup import show_markup\n",
    "from itertools import chain\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "import heapq\n",
    "%matplotlib inline\n",
    "from termcolor import colored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'utils' from '/home/jovyan/work/_Headline_generation/News_Headline_Generation/utils.py'>"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import utils\n",
    "import imp\n",
    "imp.reload(utils)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1003869it [00:56, 17722.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 50.8 s, sys: 6.42 s, total: 57.2 s\n",
      "Wall time: 56.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "texts, titles = utils.load_data('/home/jovyan/work/_Headline_generation/ria.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pool = Pool(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1003869/1003869 [01:34<00:00, 10619.58it/s]\n"
     ]
    }
   ],
   "source": [
    "parsed = pool.map(utils.parse_source, tqdm(texts))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Первые предложения"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1003869/1003869 [00:54<00:00, 18545.54it/s]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = utils.FirstSentenceTokenizer()\n",
    "first_sentences = [tokenizer.tokenize(_) for _ in tqdm(parsed)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lengths = [len(_) for _ in first_sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEACAYAAAC9Gb03AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAADQ9JREFUeJzt3d+rnVV+x/H3pw5OwYqFVnoRDVoSZXJX2CjMlRfONJJGBzvMGLyxhAQHMtdNYf6AzE0vZKT2DJV0bpQgMpNTM3gxYJ0BLxLnSicIQVI8CDXWIrQUxM63F4nleDg/9j7Ps88+++v7BYJ77f08z9fF5sNy7XXWSlUhSerrDxZdgCRpvgx6SWrOoJek5gx6SWrOoJek5gx6SWrOoJek5gx6SWrOoJek5gx6SWrua4t8eJLjwPE777zz1AMPPLDIUiRp6bz99tsfV9XdO30u+2Gvm8lkUleuXFl0GZK0VJK8XVWTnT630KmbJMeTrHz66aeLLEOSWlto0FfValWdvuuuuxZZhiS15o+xktScUzeS1JxTN5LUnFM3ktScQS9Jze2LP5g6dOjQIsuQltZ9Z18bfI/r546NUIn2M+foJak5p24kqTmDXpKacx29JDXnHL0kNefUjSQ1Z9BLUnMGvSQ15x9MSQs0xh88STvxx1hJas6pG0lqzqCXpOYMeklqzqCXpOYMeklqzr1uJKk5l1dKUnNO3UhScwa9JDVn0EtScwa9JDVn0EtScwa9JDVn0EtSc6MHfZJHkvw6yQtJHhn7/pKk2UwV9EleTPJRknc2tB9N8l6Sa0nO3mou4L+APwTWxi1XkjSraU+YOg/8BPjZFw1JbgOeB77FzUC/nOQi8Ouq+tckfwb8PfD0qBVLGtXQU66unzs2UiWal6lG9FX1JvDJhuaHgGtV9X5VfQa8DDxRVb+/9f5/Al8frVJJ0q4MOTP2APDButdrwMNJngT+Evhjbv5fwKaSnAZOAxw8eHBAGZKk7QwJ+mzSVlX1KvDqThdX1QqwAjCZTGpAHZKkbQxZdbMG3Lvu9T3Ah7PcwG2KJWn+hgT9ZeBwkvuT3A48BVyc5QZuUyxJ8zft8sqXgLeAB5OsJTlZVZ8DZ4DXgavAhap6d5aHO6KXpPmbao6+qk5s0X4JuLTbh1fVKrA6mUxO7fYekqTtDfkxVvrKG7oGXdoLnhkrSc15ZqwkNefulZLUnFM3ktScUzeS1JxTN5LUnFM3ktScUzeS1JxTN5LUnEEvSc05Ry9JzTlHL0nNOXUjSc0Z9JLUnEEvSc0Z9JLUnKtuJKk5V91IUnNO3UhScwa9JDVn0EtSc19bdAHSIt139rVFlyDNnSN6SWrO5ZWS1JzLKyWpOaduJKk5g16SmjPoJak5g16SmjPoJak5g16SmjPoJam5uQR9kjuSvJ3kr+Zxf0nS9KYK+iQvJvkoyTsb2o8meS/JtSRn1731t8CFMQuVJO3OtCP688DR9Q1JbgOeBx4DjgAnkhxJ8ijwO+DfR6xTkrRLU+1eWVVvJrlvQ/NDwLWqeh8gycvAE8AfAXdwM/z/J8mlqvr9aBVLkmYyZJviA8AH616vAQ9X1RmAJM8AH28V8klOA6cBDh48OKAMSdJ2hvwYm03a6v//pep8Vf3LVhdX1UpVTapqcvfddw8oQ5K0nSFBvwbcu+71PcCHs9zAbYolaf6GBP1l4HCS+5PcDjwFXJzlBm5TLEnzN+3yypeAt4AHk6wlOVlVnwNngNeBq8CFqnp3loc7opek+Zt21c2JLdovAZd2+/CqWgVWJ5PJqd3eQ5K0PQ8H11LzcG9pZwsN+iTHgeOHDh1aZBlaIINamj/PjJWk5hYa9P4YK0nz54hekprzx1gN4hy7tP958IgkNeccvSQ15xy9JDXn1I0kNWfQS1JzztFLUnMLXV7ppmbS8hu6xPb6uWMjVaKtOHUjSc0Z9JLUnEEvSc35Y6wkNecfTElSc07dSFJzBr0kNWfQS1JzBr0kNWfQS1JzBr0kNec6eklqznX0ktScUzeS1JxBL0nNGfSS1JxBL0nNGfSS1JxBL0nNLfTMWC3e0PM+Je1/o4/ok3wjyQtJXknyg7HvL0mazVRBn+TFJB8leWdD+9Ek7yW5luQsQFVdrapnge8Bk/FLliTNYtoR/Xng6PqGJLcBzwOPAUeAE0mO3HrvceA3wK9Gq1SStCtTBX1VvQl8sqH5IeBaVb1fVZ8BLwNP3Pr8xar6JvD0mMVKkmY35MfYA8AH616vAQ8neQR4Evg6cGmri5OcBk4DHDx4cEAZkqTtDAn6bNJWVfUG8MZOF1fVCrACMJlMakAdkqRtDFl1swbcu+71PcCHs9zAbYolaf6GBP1l4HCS+5PcDjwFXJzlBm5TLEnzN+3yypeAt4AHk6wlOVlVnwNngNeBq8CFqnp3loc7opek+Ztqjr6qTmzRfoltfnCd4r6rwOpkMjm123tIkrbnUYKS1JxHCUpSc25qtuTclEzSThYa9EmOA8cPHTq0yDIkLdDQwcr1c8dGqqQvp24kqTkPHpGk5lx1I0nNOXUjSc05dSNJzRn0ktScQS9JzfljrCQ154+xktScWyAsmFsYSJo35+glqTmDXpKa88dYSWrOH2MlqTmnbiSpOYNekpoz6CWpOYNekpoz6CWpOZdXSlJzLq+UpOacupGk5gx6SWrO3SslLbWhO8BeP3dspEr2L0f0ktScQS9JzRn0ktScQS9Jzc0l6JN8J8lPk/wiybfn8QxJ0nSmDvokLyb5KMk7G9qPJnkvybUkZwGq6udVdQp4Bvj+qBVLkmYyy4j+PHB0fUOS24DngceAI8CJJEfWfeRHt96XJC3I1EFfVW8Cn2xofgi4VlXvV9VnwMvAE7npx8Avq+q345UrSZrV0Dn6A8AH616v3Wr7IfAo8N0kz252YZLTSa4kuXLjxo2BZUiStjL0L2OzSVtV1XPAc9tdWFUrwArAZDKpgXVIkrYwdES/Bty77vU9wIfTXuw2xZI0f0OD/jJwOMn9SW4HngIuTnux2xRL0vzNsrzyJeAt4MEka0lOVtXnwBngdeAqcKGq3p3hno7oJWnOpp6jr6oTW7RfAi7t5uFVtQqsTiaTU7u5XpK0M7dAkKTmPDNWkprzzFhJas6pG0lqzqkbSWrOqRtJas6pG0lqzqkbSWrOqRtJam7o7pVfefedfW3RJUjStpyjl6TmnKOXpOaco5ek5py6kaTmDHpJas5VN5K+0oaunLt+7thIlcyPI3pJas5VN5LUnKtuJKk5p24kqTmDXpKa+8qvunGvGkndOaKXpOYMeklqzuWVktScyyslqTmnbiSpOYNekpoz6CWpuaVfR+86eEnaniN6SWrOoJek5gx6SWpu9KBP8udJ/inJK2PfW5I0u6mCPsmLST5K8s6G9qNJ3ktyLclZgKp6v6pOzqNYSdLspl11cx74CfCzLxqS3AY8D3wLWAMuJ7lYVb8bu0hJ2q+W4czZqUb0VfUm8MmG5oeAa7dG8J8BLwNPjFyfJGmgIXP0B4AP1r1eAw4k+ZMkLwB/keTvtro4yekkV5JcuXHjxoAyJEnbGfIHU9mkrarqP4Bnd7q4qlaAFYDJZFID6pAkbWPIiH4NuHfd63uAD2e5gdsUS9L8DQn6y8DhJPcnuR14Crg4yw3cpliS5m/a5ZUvAW8BDyZZS3Kyqj4HzgCvA1eBC1X17iwPd0QvSfM31Rx9VZ3Yov0ScGm3D6+qVWB1Mpmc2u09JEnb8yhBSWrOowQlqblULX5lY5IbwL8BdwEbh/cb2za+/lPg47kWuPlz53HtTp/b7v2t3tuvfboX/TnNZ2ft0/3an5s9dx7X+h0d/9ohfXq4qnYeKVfVvvkHWNmpbZPXVxZV29jX7vS57d7f6r392qd70Z/z6NP92p971ad+R8e/dh59uvGf/bZN8eoUbZt9Zi8Mee601+70ue3e3+q9/dqne9Gf03x21j7dr/059Ll+R3euYV7XzqNPv2RfTN0MkeRKVU0WXUcn9um47M/x2aez2W8j+t1YWXQBDdmn47I/x2efzmDpR/SSpO11GNFLkrZh0EtScwa9JDXXLuiT3JHkn5P8NMnTi65n2XnY+/iSfOfW9/MXSb696Ho6SPKNJC8keSXJDxZdz36zFEE/y+HkwJPAK1V1Cnh8z4tdAh72Pr4Z+/Tnt76fzwDfX0C5S2HGPr1aVc8C3wNcdrnBUgQ9Nw8nP7q+Yd3h5I8BR4ATSY5w8wCUL444/N89rHGZnGf6/tR0zjN7n/7o1vva3Hlm6NMkjwO/AX61t2Xuf0sR9DXb4eRr3Ax7WJL/vr02Y39qCrP0aW76MfDLqvrtXte6LGb9nlbVxar6JuCU7QbLHISbHk4OvAr8dZJ/YHF/ir6MBh32rk1t9R39IfAo8N0kO56vrC/Z6nv6SJLnkvwjA87I6GrI4eCLttXh5P8N/M1eF9PAoMPetamt+vQ54Lm9LqaJrfr0DeCNvS1leSzziH7w4eT6EvtzfPbp+OzTXVjmoB98OLm+xP4cn306Pvt0F5Yi6Od1OPlXlf05Pvt0fPbpeNzUTJKaW4oRvSRp9wx6SWrOoJek5gx6SWrOoJek5gx6SWrOoJek5gx6SWrOoJek5v4Pvu7rZvZkQtYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(lengths, bins=np.logspace(0, 3.6, 20))\n",
    "plt.gca().set_xscale('log')\n",
    "plt.gca().set_yscale('log')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Получение словаря"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "regex_word = re.compile('[а-яёa-z\\-]+')\n",
    "freq = Counter()\n",
    "for sentence in tqdm(first_sentences):\n",
    "    for word in regex_word.findall(sentence):\n",
    "        freq[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "freq.most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(list(freq.values()), bins=np.logspace(0, 6.3, 20))\n",
    "plt.gca().set_xscale('log')\n",
    "plt.gca().set_yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words = sum(freq.values())\n",
    "top_words = sum(_[1] for _ in freq.most_common(50000))\n",
    "top_words / all_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Получение скоров без фильтрации редких слов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rouge import Rouge \n",
    "scorer = Rouge()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_sentences = [first_sentences[i] if len(first_sentences[i]) > 3 else titles[i] for i in range(len(first_sentences)) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(len(_) for _ in titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = [titles[i] if len(titles[i]) > 5 else 'а'*20 for i in range(len(titles)) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lengths = [len(_) for _ in first_sentences]\n",
    "min(lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7min 55s, sys: 2.09 s, total: 7min 57s\n",
      "Wall time: 7min 56s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'rouge-1': {'f': 0.2486609340575932,\n",
       "  'p': 0.1749548838258469,\n",
       "  'r': 0.4695426866963995},\n",
       " 'rouge-2': {'f': 0.11076440032561513,\n",
       "  'p': 0.0766843257758511,\n",
       "  'r': 0.22291123126761683},\n",
       " 'rouge-l': {'f': 0.17336727901459906,\n",
       "  'p': 0.1606306228037659,\n",
       "  'r': 0.4301897932667798}}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "scorer.get_scores(first_sentences, titles, avg=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.17759753779926912"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(0.2486609340575932+0.11076440032561513+0.17336727901459906)/3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'до 7 февраля - того дня, когда граждане украины решат, кто на ближайшие пять лет возьмет бразды правления страной в свои руки, - осталась ровно неделя'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_sentences[2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Получение скоров с фильтрацией редких слов"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Получим словарь"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2007738it [00:45, 44219.59it/s]\n"
     ]
    }
   ],
   "source": [
    "vocab = utils.get_vocab(chain(titles, first_sentences))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Отфильтруем самые частые слова"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_top_frequent = utils.filter_vocab(vocab, n=50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Получим заголовки и первые предложения без знаков препинания и редких слов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1003869/1003869 [00:08<00:00, 121938.86it/s]\n",
      "100%|██████████| 1003869/1003869 [00:23<00:00, 42232.16it/s]\n"
     ]
    }
   ],
   "source": [
    "titles_filtered = utils.filter_texts(titles, vocab_top_frequent)\n",
    "first_sentences_filtered = utils.filter_texts(first_sentences, vocab_top_frequent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles_filtered = [titles_filtered[i] if len(titles_filtered[i]) > 5 else 'а'*20 for i in range(len(titles_filtered)) ]\n",
    "first_sentences_filtered = [first_sentences_filtered[i] if len(first_sentences_filtered[i]) > 3 else titles_filtered[i] for i in range(len(first_sentences_filtered)) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6min 57s, sys: 1.51 s, total: 6min 59s\n",
      "Wall time: 6min 58s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'rouge-1': {'f': 0.2842236396665328,\n",
       "  'p': 0.2000193040853728,\n",
       "  'r': 0.5414221130501112},\n",
       " 'rouge-2': {'f': 0.12612636548805686,\n",
       "  'p': 0.08720967178968597,\n",
       "  'r': 0.257567061552075},\n",
       " 'rouge-l': {'f': 0.19571938456778085,\n",
       "  'p': 0.18177296712670088,\n",
       "  'r': 0.4906582209423502}}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "scorer.get_scores(first_sentences_filtered, titles_filtered, avg=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20202312990745683"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(0.2842236396665328+0.12612636548805686+0.19571938456778085)/3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Получение скоров с сохранением пунктуации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1003869/1003869 [00:15<00:00, 63689.27it/s]\n",
      "100%|██████████| 1003869/1003869 [00:40<00:00, 24493.04it/s]\n"
     ]
    }
   ],
   "source": [
    "titles_filtered = utils.filter_texts_with_punctuation_spaces(titles, vocab_top_frequent)\n",
    "first_sentences_filtered = utils.filter_texts_with_punctuation_spaces(first_sentences, vocab_top_frequent)\n",
    "titles_filtered = [titles_filtered[i] if len(titles_filtered[i]) > 5 else 'а'*20 for i in range(len(titles_filtered)) ]\n",
    "first_sentences_filtered = [first_sentences_filtered[i] if len(first_sentences_filtered[i]) > 3 else titles_filtered[i] for i in range(len(first_sentences_filtered)) ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7min 33s, sys: 1.77 s, total: 7min 34s\n",
      "Wall time: 7min 33s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'rouge-1': {'f': 0.25675646704517774,\n",
       "  'p': 0.18067755018686546,\n",
       "  'r': 0.49011926614274137},\n",
       " 'rouge-2': {'f': 0.11654683276540687,\n",
       "  'p': 0.08114441280707248,\n",
       "  'r': 0.23452119002988178},\n",
       " 'rouge-l': {'f': 0.17894743541237113,\n",
       "  'p': 0.16617113221318203,\n",
       "  'r': 0.44973780457592655}}"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "scorer.get_scores(first_sentences_filtered, titles_filtered, avg=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Получим также скоры для каждого предложения отдельно и найдем наихудшие"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1003869/1003869 [08:05<00:00, 2066.95it/s]\n"
     ]
    }
   ],
   "source": [
    "scores_single = [scorer.get_scores(first_sentences_filtered[i], titles_filtered[i]) for i in trange(len(first_sentences_filtered))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(len(_) for _ in scores_single)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.41619560796253974"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_single[0][0]['rouge-1']['f'] + scores_single[0][0]['rouge-2']['f'] + scores_single[0][0]['rouge-l']['f']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_f1(x):\n",
    "    return (x[1][0]['rouge-1']['f'] + x[1][0]['rouge-2']['f'] + x[1][0]['rouge-l']['f']) / 3\n",
    "worst = heapq.nsmallest(10000, enumerate(scores_single), key=mean_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Средняя F-мера: 0.0\n",
      "\u001b[32mПервое предложение: do tej pory płaczę myśląc o tej tragedii\u001b[0m\n",
      "Отфильтрованное первое предложение: do tej p my l c o tej tragedii\n",
      "\u001b[32mЗаголовок: аааааааааааааааааааа\u001b[0m\n",
      "Отфильтрованный заголовок: аааааааааааааааааааа\n",
      "\u001b[34mТекст без тегов:\n",
      " do tej pory płaczę myśląc o tej tragedii. szok i przerażenie pierwszych godzin zastąpiły żal i współczucie. przyjmijcie nasze kondolencje. i niech pan bóg da naszym krajom pokój i wzajemne zrozumienie!\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "j = np.random.choice(len(worst))\n",
    "i = worst[j][0]\n",
    "f1 = mean_f1(worst[j])\n",
    "print('Средняя F-мера:', f1)\n",
    "print(colored(f'Первое предложение: {first_sentences[i]}', color='green'))\n",
    "print('Отфильтрованное первое предложение:', first_sentences_filtered[i])\n",
    "print(colored(f'Заголовок: {titles[i]}', color='green'))\n",
    "print('Отфильтрованный заголовок:', titles_filtered[i])\n",
    "\n",
    "print(colored(f'Текст без тегов:\\n {parsed[i]}', color='blue'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "best = heapq.nlargest(10000, enumerate(scores_single), key=mean_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Средняя F-мера: 0.7414874595859814\n",
      "\u001b[32mПервое предложение: сорок человек эвакуировали в уфе из-за пожара на крыше жилого дома, сообщается на сайте мчс рф\u001b[0m\n",
      "Отфильтрованное первое предложение: сорок человек эвакуировали в уфе из-за пожара на крыше жилого дома, сообщается на сайте мчс рф\n",
      "\u001b[32mЗаголовок: сорок человек эвакуировали в уфе из-за пожара на крыше жилого дома\u001b[0m\n",
      "Отфильтрованный заголовок: сорок человек эвакуировали в уфе из-за пожара на крыше жилого дома\n",
      "\u001b[34mТекст без тегов:\n",
      " \n",
      "москва, 28 дек — риа новости. сорок человек эвакуировали в уфе из-за пожара на крыше жилого дома, сообщается на сайте мчс рф.\n",
      "информация о возгорании на крыше дома 34 по улице султанова поступила в четверг в 21.33 мск. спустя час огонь был локализован на площади 600 квадратных метров. спасатели эвакуировали 40 жильцов.\n",
      "\"в 00.53 мск 28 декабря пожар ликвидирован. погибших и пострадавших нет\", — говорится на сайте.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "j = np.random.choice(len(best))\n",
    "i = best[j][0]\n",
    "f1 = mean_f1(best[j])\n",
    "print('Средняя F-мера:', f1)\n",
    "print(colored(f'Первое предложение: {first_sentences[i]}', color='green'))\n",
    "print('Отфильтрованное первое предложение:', first_sentences_filtered[i])\n",
    "print(colored(f'Заголовок: {titles[i]}', color='green'))\n",
    "print('Отфильтрованный заголовок:', titles_filtered[i])\n",
    "\n",
    "print(colored(f'Текст без тегов:\\n {parsed[i]}', color='blue'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Типичные причины низких скоров:\n",
    "* Первое предложение распознано неправильно из-за ошибок в тексте. Часто в заметках есть типовое предложение вида `петропавловск-камчатский, 7 янв - риа новости`. Его мы пропускаем и в качестве гипотезы берем следующее предложение. Регулярка, который мы ищем это типовое предложение, срабатывает не всегда. Например, `петропавловск-камчатский, 7 янв- риа новости` - здесь должен быть пробел перед тире. Также было, что после названия города стоит точка, а не запятая.\n",
    "* Первое предложение заметно длиннее заголовка\n",
    "* Есть тексты, отличающиеся от типичных новостных заметок (записи в блоге, аналитика, письма читателей). Они написаны более свободным стилем, и первое предложение может не содержать обобщения статьи."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Проанализируем, как метрика зависит от разности длин"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "len_diff = [abs(len(x) - len(y)) + 1 for x, y in zip(first_sentences_filtered, titles_filtered)]\n",
    "f1 = [mean_f1(_) for _ in enumerate(scores_single)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f6db8af17f0>"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAG4BJREFUeJzt3X+QHOV95/H3d5cRrDDHStE6hVbCEj5FiYgcC/ZAKV2l/FsCKpIO/wLjii/nMnV35u6Mc6oSZQoLjisrpuzkUsfFhxMq/sGBAJONfCglXBGuXFGWYLmVEIJTEAIkjVyWYrScD8bWsvu9P6ZH9M52z/Ts9mzPdH9eVVs709Pb8+2d2c/0Pv3085i7IyIi+dKTdQEiIpI+hbuISA4p3EVEckjhLiKSQwp3EZEcUriLiOSQwl1EJIcU7iIiOaRwFxHJofOyeuJFixb5smXLsnp6EZGu9Oyzz/6juw80Wy+zcF+2bBkjIyNZPb2ISFcys9eSrKdmGRGRHFK4i4jkkMJdRCSHFO4iIjmkcBcRySGFu4hIDincRURyqGm4m9n9ZnbKzJ6PedzM7M/M7IiZPWdmV6RfpoiItCLJRUx/BfxX4Lsxj18DrAi+rgb+PPieumVbH5+27NXt17XjqUREulrTI3d3/3vg9QarbAK+61V7gX4zuyStAmuigr3RchGRIkujzX0QOB66fyJYJiIiGUkj3C1imUeuaHazmY2Y2cjp06dTeGoREYmSRrifAJaG7i8BTkat6O73ufuQuw8NDDQd1ExERGYojXDfCfxB0GtmLfCGu/80he2KiMgMNe0tY2YPAh8AFpnZCeCrQAnA3b8F7AKuBY4AbwF/2K5iRUQkmabh7u43NnncgS+mVpGIiMxaLq5QvenbP8m6BBGRjpKLcH/q5Ubd8EVEiicX4Q4wPFrOugQRkY6Rm3Df8sgBBbyISCA34T4+6dyz+3DWZYiIdITchDvAybFK1iWIiHSEXIV7//xS1iWIiHSEXIW7R45oIyJSPLkK9zcq41mXICLSEXIV7ov7+7IuQUSkI+Qq3LesX5l1CSIiHSFX4b55jeYIERGBnIW7iIhU5SrcdYWqiEhVrsJ9285DWZcgItIRchXuY+oKKSIC5CzcRUSkSuEuIpJDCncRkRxSuIuI5FDuwv324YNZlyAikrnchfsDe4+pv7uIFF7uwt1BMzKJSOHlLtwBypqRSUQKLpfh3muWdQkiIpnKZbhPaEomESm4XIb7oCbtEJGCy2W4a9IOESm6XIa7Ju0QkaLLZbiLiBRdonA3sw1mdtjMjpjZ1ojHLzWzJ81s1MyeM7Nr0y9VRESSahruZtYL3AtcA6wCbjSzVXWr3Q487O5rgBuA/5Z2oa3QEAQiUnRJjtyvAo64+1F3Pws8BGyqW8eBfxLcvhg4mV6Jrfv+3mNZPr2ISOaShPsgcDx0/0SwLGwb8FkzOwHsAv5d1IbM7GYzGzGzkdOnT8+gXBERSSJJuEdd7ll/ldCNwF+5+xLgWuB7ZjZt2+5+n7sPufvQwMBA69WKiEgiScL9BLA0dH8J05tdPg88DODuPwEuABalUaCIiLQuSbg/A6wws+VmNo/qCdOddescAz4MYGa/RTXc1e4iIpKRpuHu7m8DtwC7gRep9oo5ZGZ3mdnGYLU/Ar5gZgeAB4F/6Z7tAC/qMSMiRXZekpXcfRfVE6XhZXeEbr8ArEu3tNn5/t5j3L15ddZliIhkItdXqGpGJhEpqlyH+22PHVTAi0gh5TrcK+MTfGnHftZt36OQF5FCyXW415THKjqKF5FCKUS4Q/UoXhNni0hRFCbcAU5q4mwRKYhChftiTb8nIgVRqHDX9HsiUhSFCndNvyciRVGYcO+1qMEtRUTyqTDhftnA/KxLEBGZM4UJ96On38q6BBGROVOYcJ/IdpBKEZE5VZhwV4u7iBRJYcK9p8c0/ICIFEZhwn1i0jX8gIgURmHCHaoDiImIFEGhwl193UWkKAoV7uoxIyJFUahwFxEpCoW7iEgOFSrce9TkLiIFUahwR03uIlIQhQr3yawLEBGZI+dlXcBcW7d9DyfHKizu72PL+pUa411Ecqlw4V67kKk8VuG2xw4CmsRDRPKnUM0y9SrjExqSQERyqdDhDnBSQxKISA4VPtwX9/dlXYKISOoShbuZbTCzw2Z2xMy2xqzzKTN7wcwOmdn/SLfM9ugr9bJl/cqsyxARSV3TE6pm1gvcC3wUOAE8Y2Y73f2F0DorgNuAde5+xsze3a6C02Cg3jIikmtJestcBRxx96MAZvYQsAl4IbTOF4B73f0MgLufSrvQNL2y/bqsSxARaaskzTKDwPHQ/RPBsrDfAH7DzJ4ys71mtiGtAttBMzKJSN4lOXKPGpGl/kL+84AVwAeAJcD/MrPfdvexKRsyuxm4GeDSSy9tudi0qH+7iORdkiP3E8DS0P0lwMmIdf7G3cfd/RXgMNWwn8Ld73P3IXcfGhgYmGnNs6b+7SKSd0nC/RlghZktN7N5wA3Azrp1hoEPApjZIqrNNEfTLDRt6t8uInnWNNzd/W3gFmA38CLwsLsfMrO7zGxjsNpu4Odm9gLwJLDF3X/erqLToP7tIpJnicaWcfddwK66ZXeEbjvw5eCrK6h/u4jkWWGvUNXJVBHJs8KGu7pDikieFTbcv7RjvwJeRHKrsOEO1f7uCngRyaNCh7v6u4tIXhU63EH93UUknwof7j1mLN/6OOu271ETjYjkRuHmUK034dVhcjSnqojkSeGP3MPUBi8ieaFwr6M2eBHJA4V7HY05IyJ5oHCvozFnRCQPFO51dDJVRPJA4R7S31fKugQRkVQo3EO2bbw86xJERFKhcA9Rk4yI5IXCXUQkhxTuIiI5pHAP/PpF87IuQUQkNQr3wM9+cVYDh4lIbijcQ2577LmsSxARSYXCPaQyPpl1CSIiqVC4i4jkkMK9zjJN3CEiOaBwj1CbuEMBLyLdSuEeQxN3iEg3U7g3UNbEHSLSpRTuDRioaUZEupLCvQEHNc2ISFdSuDehOVVFpBslCncz22Bmh83siJltbbDeJ8zMzWwovRKzpTlVRaQbNQ13M+sF7gWuAVYBN5rZqoj1LgL+PbAv7SKzVB6rqN+7iHSdJEfuVwFH3P2ou58FHgI2Raz3n4CvA79Msb6OUB6r8OUd+xXwItI1koT7IHA8dP9EsOwcM1sDLHX3/9loQ2Z2s5mNmNnI6dOnWy42S5NoYDER6R5Jwt0ilvm5B816gD8B/qjZhtz9PncfcvehgYGB5FV2CA0sJiLdIkm4nwCWhu4vAU6G7l8E/DbwYzN7FVgL7MzTSdWw5Rp7RkS6QJJwfwZYYWbLzWwecAOws/agu7/h7ovcfZm7LwP2AhvdfaQtFWfM0dgzItL5moa7u78N3ALsBl4EHnb3Q2Z2l5ltbHeBnUpjz4hIJzsvyUruvgvYVbfsjph1PzD7srpDeazC8q2Ps7i/jy3rV7J5zWDkesOjZe7ZfZiTY5Wm64qIpCFRuEu8cDMNMC20h0fL3PbYQSrjE9BkXRGRtGj4gZTENdPcs/vwuWBvtq6ISFoU7imKGocmbmwajVkjIu2kcE9R1Dg0cWPTaMwaEWknhXtKDNiyfuW05VvWr6Sv1DtlWV+pN3JdEZG06IRqCgy4ae2lkSdIa8vUW0ZE5pLCfZYGE4T15jWDCnMRmVMK91l6auuHsi5BRGQatbnP0ke/+eOsSxARmUbhPksvnXpTAS8iHUfhnoKXTr2pkSJFpKMo3FNSHqtw64793D58MOtSREQU7mly4IG9x3QELyKZU2+ZlDnVPu1RXR9vHz7Ig/uOM+FOrxk3Xr2UuzevnvsiRST3FO5tEDVuzE3f/glPvfz6ufsT7nx/7zEABbyIpE7NMm3QP7805f7waHlKsIc9uO945HIRkdlQuLfBG5XxKe3ujYb3nXCPfUxEZKYU7m0w6VMDvdHwvr1mc1GSiBSMwr1NyqFAv6AU/2tee9mCuShHRApGJ1TbaNnWx1nx7gupjE/GrvP0q2cYHi1rYDERSZWO3NvspVNvNnx8fMI15Z6IpE5H7h2gPFZhzV1PcOatcQD6+0ps23i5juZFZMZ05N4hasEOMFYZ58s79utKVxGZMYV7h5oEtu08lHUZItKlFO4dbKwy3nwlEZEIanOXSMOjZc37KtLFFO4dbEEwjMFcB+3waJnbHjtIZXwCqJ7wve2x6lDGCniR7qBw72DXve8S3n/nE1OaZ+YiaO/ZffhcsNdUxidiR7sUkc6jNvcOtuPp45Ht7rWgbZe44RIaDaMgIp0lUbib2QYzO2xmR8xsa8TjXzazF8zsOTP7OzN7T/qlFs/4ZPygYu0M2sX9fS0tF5HO0zTczawXuBe4BlgF3Ghmq+pWGwWG3P19wKPA19MuVKZqZ9BuWb+SvlLvlGV9pV62rF/ZtucUkXQlOXK/Cjji7kfd/SzwELApvIK7P+nubwV39wJL0i1TwtodtJvXDPK161cz2N+HAYP9fXzt+tVqbxfpIklOqA4C4RklTgBXN1j/88DfzqYoiddj8PErB9setJvXtP85RKR9khy5Rw04HtkYbGafBYaAe2Iev9nMRsxs5PTp08mrlHMmHX7wbFlDE4hIQ0nC/QSwNHR/CXCyfiUz+wjwFWCju/8qakPufp+7D7n70MDAwEzqFdLrLTM8Wmbd9j0s3/o467bv0QeGSI4kaZZ5BlhhZsuBMnAD8JnwCma2BvjvwAZ3P5V6lTJNeazCuu17zl3Q1OqFTnm7UElX1IpM1TTc3f1tM7sF2A30Ave7+yEzuwsYcfedVJth3gU8YtVp4465+8Y21i28E8iPjBybMgF3eazClkcPAPFBnacLlfL2QSWShkRXqLr7LmBX3bI7Qrc/knJdklBlfGJKsNeMTzh3/vBQbLjl6UKlPH1QiaRFww/k2Jm3xqcMX7Bgfomv/n51EpDF/X1T5nmt6cYLlfL0QSWSFoV7zoWHLzjz1vi55pot61dOacqAqf3nu6kNO08fVCJp0dgyBVObs7XRhUq1NuzyWAXnnTbsTu1NoytqRabTkXsB1Zor4i5U6rY27FpN3fKfhshcULgXUH8wTnycqCaO2vJ12/d0ZIDqilqRqdQsU0C/rDsqr9drURclV4Wbam7dsZ/bhw+mXJ2IpEFH7gVUGZ9keLQce/HThMcPNRzmwAN7jzH0noU6ahbpMOYJ/5DTNjQ05CMjI4nXX7b18TZWU0z9fSXePPs24xNT3wNGzOBBMQb7+3hq64dSrU1EopnZs+4+1Gw9HbkXWNQsT9BasIP6k4t0IoW7zFqPGcu3Pj7lJGujfvLd1IdepFsp3GXWam30tf7wI6+9zg+eLUeO9QLEjgMD6s4okhaFu6SqMj7Bg/uOTzspGx6mOKoP/a0P7yf8I40G/9KRv0hz6gopqYvrbXNyrBLbPh/1I1Hj1nfb1bMiWVG4S2KlWb5bnNmfrG109ayIvEPhLomNT879c9YP/hV35F8eq2hGKZEQhbt0rKjBvy7uix86Qc00Iu/QCVXpSD0GH79ykHt2H+bWHftZ3N/HB39zgF/86u2mP9vJg5yJzBWFu3SkSYcdTx9nfPKdbpYP7D2WuM2+1nwT1bMG1OVS8k/hLh2rFuw1rZyMXdzfFzm36pZHD4Az5UOj1jf/yf9zumMDX90/pVUKd8mdWlt9VM+a+nF0oNqME/6vIK0JttMKZE0ALjOhE6oyZ/r7SpR644cTTkN4RqlWxrypj/zZdq8cHi2z5ZEDU/rjb3nkwIxO9Kr7p8yEjtxlTvT2GGbRR85J9ZV6+fiVg3x/77HIxw2mjE4ZN7dqUrOZnGTbzkPTmpXGJ51bH95/7gRx0u1pAnCZCR25y5yY12uceSt6FMpG+ko9U+Z4vXtzdd7XKOE+8cOjZd6M6VnTU/fPQ6P/JWZ6JWzsiJveepfNuIm+NQG4NKJwlzlRmeEVUAsvPJ9Xtl/HU1s/dO4ot9GE2MOjZdbc9QRf2rE/NmBxWDC/dO5D46a1l07bXvQ+pNsUUhmf4M4fHmq6XhEmAB8eLbNu+x5diJYiNctIRyuPVVhz1xPnjvr7+0ps23g5X7t+dWQXx/CJxziTVI+gX9l+3bllQ+9ZyD27DzdtxknaFLJgfinRfypn3ho/NytWnLxPAK4Txu3RNTMx3fTtn/DUy6+3sSLpFqUe455P/s60P/x12/e01Mb+aijca5Zvfbxhl8uks04Nj5bZ8uiBROcYij6TVdzrVvTfS5ykMzF1TbPMA1/4Xda9d2HWZUjKkjSH1Buf9MjmkTROMDZqx26lKWTzmkHu+cTvMNjfh1H9jyNO0U+M6oRxe3RVs8wDX/hdAD76zR/z0qk3M65G0tCo90sj5bHKtH7kF/eV4tvZ6yyYHx22W9avjGzaWTC/xFd///KWmgk2rxmcMvtU/Zj1NUU/MRrXq6nov5fZ6ppmmSiaNFtmorfH+EZEs87waJk7f3hoSlv5hfN6KfX28EZlfMZt3fVtymF9pV6+dv1qgCnPXTu3kMc25/oP5Q/+5sCUmbvgnd/LbPc/j1f2Jm2W6epwB7XFS+t6gG9++v1T/siTtpHX2vsh+gRnVJjEnajtNeMbn6puq9FzD8aEUvi5+ueXcGfah1DScEs7BKO2N/La65HjA5V6jU//s6UtD/8QV3NteXmsgjH1ArXa/bjfaSvPMxNpbCvVcDezDcB/AXqBv3D37XWPnw98F7gS+DnwaXd/tdE2ZxLuOlKXtNSfrGvlZGxfqQewaUeaH79ycNoRaH24hBnVHjtJnrv+SLbRfwON6ok6Io7a1myOnKO210O1l1KcBfNLjN7xsVk9R9w+x0myj2n+btLaVmonVM2sF7gXuAZYBdxoZqvqVvs8cMbd/ynwJ8AfJ640IQW7pKn+ZF0rJ+8q45ORwwE8uO/4tOWNDp1qbcpJnru+j33UkARJ6onqq5/28AZR22t2lUOrF7jF1Ry1z3GS7GOav5u5HkYiSW+Zq4Aj7n7U3c8CDwGb6tbZBHwnuP0o8GEza+8gIiKzUH+yLo2Td3Fzx0YJ97xJ+tzhD4EkHwiN5rJtdL+V50jz59J4jlZeg0bbafb4TPZxrnsFJQn3QeB46P6JYFnkOu7+NvAG8Gv1GzKzm81sxMxGTp8+PbOKRWap1GPTujS2crVn/fAFNb0Jj2fCg5vVnjvJgGrhD4EkHwhx9ST9YJvpB95Mfq5RV9FWniPpa9BsO80en8k+zvUwEknCPeq3Vf/xmGQd3P0+dx9y96GBgYEk9ckMmVVflAvntd6PfMp2qP7h1b73xiUb1XbTde9d2HCslij9fSU+u/ZSwn+XpZ747oqlXmPFuy9MtO0L5/UyPzSzd39fKfICqM1rBvns2kubbq/Ua3zm6unDFfSVernx6qVN++3X2vrDz1/rEx+3v7Xthz+AooYkSFJPVF/9tIc3iNpeo6Ap9RjbNl4+6+eI22er+x5ev9k+pvm7methJJL0cz8BLA3dXwKcjFnnhJmdB1wMqAtLRlo9+ZZ0O7VtzfRsf9wJpVqXv7s3r479ubheEdt2HjrXt30mfdHD7t68+twwBOFuelG9OOrXq18e1VOj0R9yuE98o30Orw807S0TV2f9c4e3NdseIXHbG3ntdR7cd3xK00krvVaS1hy3zzN576b5u5nrYSSa9pYJwvofgA8DZeAZ4DPufii0zheB1e7+r83sBuB6d/9Uo+2qt0xj6967kE8OXcpX/vogb56dHsrhLl1xARQW7h7Wa8aEO/19Jd46+zZngy54RrUnSGV8sm1vvDz2O45TpH2VuZN2V8hrgT+l2hXyfnf/z2Z2FzDi7jvN7ALge8AaqkfsN7j70UbbTKufu4hIkSQN90TDD7j7LmBX3bI7Qrd/CXyy1SJFRKQ9umbgMBERSU7hLiKSQwp3EZEcUriLiOSQwl1EJIcU7iIiOaRwFxHJocwm6zCz08BrM/zxRcA/plhOFrQP2ev2+kH70Anmuv73uHvTwbkyC/fZMLORJFdodTLtQ/a6vX7QPnSCTq1fzTIiIjmkcBcRyaFuDff7si4gBdqH7HV7/aB96AQdWX9XtrmLiEhj3XrkLiIiDXRduJvZBjM7bGZHzGxr1vXEMbNXzeygme03s5Fg2UIz+5GZvRR8XxAsNzP7s2CfnjOzKzKq+X4zO2Vmz4eWtVyzmX0uWP8lM/tcB+zDNjMrB6/F/mB+gtpjtwX7cNjM1oeWZ/I+M7OlZvakmb1oZofM7D8Ey7vmdWiwD930OlxgZk+b2YFgH+4Mli83s33B73SHmc0Llp8f3D8SPL6s2b61nbt3zRfVyUJeBi4D5gEHgFVZ1xVT66vAorplXwe2Bre3An8c3L4W+FuqkyGtBfZlVPPvAVcAz8+0ZmAhcDT4viC4vSDjfdgG/MeIdVcF76HzgeXBe6s3y/cZcAlwRXD7IqqzoK3qptehwT500+tgwLuC2yVgX/D7fZjqZEQA3wL+TXD73wLfCm7fAOxotG9zsQ/dduR+FXDE3Y+6+1ngIWBTxjW1YhPwneD2d4DNoeXf9aq9QL+ZXTLXxbn73zN97ttWa14P/MjdX3f3M8CPgA3tr74qZh/ibAIecvdfufsrwBGq77HM3mfu/lN3/9/B7V8ALwKDdNHr0GAf4nTi6+Du/v+Cu6Xgy4EPAY8Gy+tfh9rr8yjwYTMz4vet7bot3AeB46H7J2j8psmSA0+Y2bNmdnOw7Nfd/adQ/QMA3h0s7+T9arXmTt2XW4Jmi/trTRp0+D4E/9qvoXrU2JWvQ90+QBe9DmbWa2b7gVNUPxxfBsbc/e2Ies7VGjz+BvBrZLgP3RbuFrGsU7v7rHP3K4BrgC+a2e81WLeb9qsmruZO3Jc/B94LvB/4KfCNYHnH7oOZvQv4AfAld/+/jVaNWNap+9BVr4O7T7j7+4ElVI+2f6tBPR23D90W7ieApaH7S4CTGdXSkLufDL6fAv6a6pvjZ7XmluD7qWD1Tt6vVmvuuH1x958Ff6iTwLd559/ijtwHMytRDcUH3P2xYHFXvQ5R+9Btr0ONu48BP6ba5t5vZrW5p8P1nKs1ePxiqs2Dme1Dt4X7M8CK4Iz1PKonLnZmXNM0ZnahmV1Uuw18DHieaq21XgufA/4muL0T+IOg58Na4I3av+AdoNWadwMfM7MFwb/dHwuWZabu/MW/oPpaQHUfbgh6OiwHVgBPk+H7LGin/UvgRXf/Zuihrnkd4vahy16HATPrD273AR+heu7gSeATwWr1r0Pt9fkEsMerZ1Tj9q395uKsbZpfVHsH/APV9q+vZF1PTI2XUT1DfgA4VKuTahvc3wEvBd8X+jtn5u8N9ukgMJRR3Q9S/Xd5nOoRx+dnUjPwr6ieODoC/GEH7MP3ghqfo/rHdklo/a8E+3AYuCbr9xnwz6n+2/4csD/4urabXocG+9BNr8P7gNGg1ueBO4Lll1EN5yPAI8D5wfILgvtHgscva7Zv7f7SFaoiIjnUbc0yIiKSgMJdRCSHFO4iIjmkcBcRySGFu4hIDincRURySOEuIpJDCncRkRz6/1/YdGS11CdsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(len_diff, f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEACAYAAABI5zaHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHXtJREFUeJzt3X2QXFV63/HvM6ORaRHC8CI71ggxWkrWrtZaozAFjvVHYF8iWCeggHeBxZVyilqK2DilhUxlFBQCBBeKVclutkJsg72F1xAhAdqxiLSRtwyUEwXJjDISY8EqK+RFmpYryCsNcaTeVTPz5I+enunpubf79tvcvrd/nyoV6tO3e85cDb8+c+65zzF3R0RE0qUr7g6IiEjzKdxFRFJI4S4ikkIKdxGRFFK4i4ikkMJdRCSFFO4iIimkcBcRSSGFu4hICincRURSaFFcX/jqq6/2/v7+uL68iEgiHTp06K/dfWm142IL9/7+fkZGRuL68iIiiWRmH0Q5TtMyIiIppHAXEUkhhbuISAop3EVEUkjhLiKSQgp3EZEUqhruZvYtM/vQzP4i5Hkzs2+a2XEze8fM/m7zuykiIrWIss79eeA/Ad8Oef42YNX0n5uA35n+b0v0D+2Z1/bDrb+c2K8jItIKVUfu7v5nwNkKh9wBfNsLDgC9ZvazzepgqaDArdTe7l9HRKRVmjHn3gecKnk8Pt0mIiIxaUb5AQto88ADzR4AHgBYsWJFE750exoezbJt3zFOT+RY1pthcMNqNq7T552ILJxmjNzHgWtKHi8HTgcd6O7PuvuAuw8sXVq17k0iDY9mGXz5CNmJHA5kJ3IMvnyE4dFs3F0TkQ7SjHDfDfyT6VUzvwh85O5/1YT3TaTHdx8lPzX3F5f8lPP47qMx9UhEOlGUpZDbgbeA1WY2bmb3m9mDZvbg9CF7gRPAceA54Ndb1dmw1SrNXsXSyNeZyOVrahcRaYWqc+7ufm+V5x34jab1qIqFWo7YLsseNX8vIvWIrZ67VDc8mmXzrjFy+UmgMH+/edcYABvX9bU0+PWhIpJsCvc2tm3fsZlgL8rlJ9m27xhAxeCPYsvwGNsPnmLSnW4z7r3pGp7auDbwQ2XwlSM8vvsoH+XyXJ7pwQwmLuQV/CJtSuEes0oj5NMTucDXnJ7IVQz+kQ/OzgvtgWuvnPN1+q/KsP/92XvTJt154cBJXj00Ti4/Ne9r5id95rpB6fWD4ofKyAdneeP7ZzTSF2kTCvcYVZt2WdabIRsQ8Mt6M6HBn53I8cKBkzOPi6Fd2padyAW+LxAY7NXk8pO8eODkzM0N9fwWUUpTQiKNU1XIGFWbdhncsJpMT/ec5zM93QxuWM2y3syC9TOK8rvWSr+PWhQ/8ErvE9i8a0z3CYjUSOEeo0rTLlAY9d51Qx/dVrgJuNuMu27oY+O6Pvqvaq9wD5KdyNUcytU+8IZHs6zf+jorh/awfuvrCn2REJqWiVGlaRcoBNmOPy/MnUNhimXHn59i4NorOXDi3IL2tV61Ts9U+sCrNo0lIrMSN3LfMjzGdZv30j+0h+s272XL8FjcXapbpWkXqHy3azHw212t0zNh003LejNVR/UiMitRI/ctw2OBFwsBntq4Nq5u1a042gy7eFjv3a5GSOW2mGQncqzf+vq87zHowunghtVzRucw+4H3tR2HA98/bLQv0skSFe7bD54KbU9iuEMh4Js9pdBOwQ6FD5vi9FPp0slXD2XnTbE8fedanr5zbeAH3rZ9xypOY4nIrESFe9hURCumKNp9OV63WSKmZoJ+i8jlJ2fW4Ze3b9t3jP1Dnw0815VG9SIyV6LC3QyC8syCKso3YHg0y+ArR8hPFr5Y8Q5NaJ8Ld59YuoQffHg+7m5UFfbxE/bBVGmKpdo0lojMSlS4ZxZ1cSHgJpvMouZeF37itaMzwV6Un3SeeO1o1SAJm+8O+/yp9zeEE2cuVD2mnYV9UIdNsZSfp6/ffb1CXaSCRIV72N2T9dxVWcm5C8EXLMPaS3V3GR9PzU+t7q758T48muWRl48wOTX7G8IjL0f7DSEJUzKVZBZ14dicKRYDbvnk/E1ctARSpHaJWgp5eaanpvY4BAV7WPuj3xmbCfaiySnn0e8UgitstN/kWahY5PJT3HXD3GB24IUDJ1n35J8wPJqduWFp047DWgIpUqNEjdzD5tabPee+UM5fnKzYHjY2T/aYvaAY5EHOXcizKWTZY6niHbAavYvMl6iR+0TItEhYu6Sf6s6IBEtUuFe6e7FdXLq4u6Z2aUwuP8mmHYdVZ0akTKLCfXDDanrKLkz2dFlbrXOuNtVSKtMTfPqL7b0h1xLC2jtZdiLH13YcVsCLTEtUuAOUr4tp7jqZxnWHXAAIan/6zs8EHltsT9s1hlZzYJMCXgRIWLg/8drRwNUlT7x2NKYezVfLXbQvjwRfUCy26xpDfaLOw6t8sKRZosK9kfXnC6UvZP4/qL10m7ug9iRcY2hHUZZJalMQSbtEhXsShG2iUc/mGkE39FRql1mVyhgMj2Z5ZOcRrZ2XVFO4N9lbJ4JH42Htlew6NF5Tu8yqVMZg866xumrbiCSJwr3JQm5QDWwPWSwz0x5UR6dSuxRUqhQZtOFHKU15SVok6g7VtPk4JKPD2kt10X4rhdpFcZ/ZotKiY5Xu7lX5YEkTjdxj1MgF0zSUIGiVVw9lZy6Mll84DdNtxtN3rlUpA0kNhXuMGrn42k7F0tpNLj/J47sLy2OrTcNAYcT+77/8Cwp2SZVETcss1GYdC+V/hiyFDGsvldTveaFM5PL0D+2peIyBNvyQ1EpUuIeVME9qafNGqj6209r+JLp0cTdHn7w17m6ItEykaRkzu9XMjpnZcTMbCnh+hZm9YWajZvaOmX2x+V2t7db+tOvE77mZzl+cpH9oz0zteJG0qRruZtYNPAPcBqwB7jWzNWWHbQF2uvs64B7gPze7o7BwG2Svv+7KmtpbJewfp4vk78TULoq14/uH9nDfc2/F3R2Rpokycr8ROO7uJ9z9IvAScEfZMQ787em/Xw6cbl4XZy3UzkRfGlhRU3upWsoPVKv6WGnaJuzraDxfv/3vn+XnHt2rkbykQpRw7wNOlTwen24r9Tjwq2Y2DuwFfrMpvSuzUDsThRUii1KgrJayxP/wF3428D2K7WH7fi/qKnydTM/cGvGZnm4tkWzQxUlXjRlJhSjhHjQYLM+Qe4Hn3X058EXgj8xs3nub2QNmNmJmI2fOnKm9twuk0QJl5fulhu2r+mpIGYFie9iNqPmpwsbQy6+4ZE57+WOpj2rMSBpECfdx4JqSx8uZP+1yP7ATwN3fAi4Bri5/I3d/1t0H3H1g6dJ0Fr/6l6++M++Tz6fby+VC0jusvdR9z73FDz48P6et/LHUTzVmJOmihPvbwCozW2lmiylcMN1ddsxJ4HMAZvYpCuHevkPzFvpJSO2AsPZ6hZULluZwoF913iXBqoa7u38MPATsA96jsCrmqJk9aWa3Tx/2CPBVMzsCbAd+zV3LOST5itv3KeglaSLdxOTueylcKC1te6zk7+8C65vbNZH2UBylFDf0AHRHq7S9RN2hGlYJUQVyZKEUL7YGhXtp9UmVNZC4JSrcw2atk1r6tpEPq0xPV6QLr9J8QRdbi9Uni0XKNMqXuGnQG6NGPqzCNgWR1usym7epdlD1yUpLKrU5t7RaokbuMqvZq28kumLph9LRedjSSY3yJS4auYs0oDg6r2XjlVpH+SL10MhdpEGnJ3J8/e7r54zGYXbbvi3DY2w/eIpJd7rNtDm3LAiFu0iDlvVmZqZTylfLjHxwlhcOnJw5tlI1T23OLc2kcBdpQGlRuI3r+ubNmT+y80ik99Hm3NJsmnMXaUB+ynl899HA1S7Do9mKI/W+3gw2/V9tzi3NppF7jMLWqmd69JmbJBO5/LzVLsUVMZXc8smlPLVxbcv7J51JKRKjS8rqsVdrl/aVy0/y+O7Zev9BK2LKvXDgJFuGK38AiNRL4R6jRuvGS3uZyOVnwjrqypftB09VP0ikDgr3GDWy4feiLm2o145ePHCS4dFs5JUv2gtXWkXhHqNGNvwO291J4uUUpmQGN6ymO+IHsEoPSCso3GN0xZLgDbLD2iUZshM5Nu04zGTED2Dt2SqtoHCPUdgAXb+pdxaVHpBWULjH6KNc8IXTsHZJL5UekGZLVLj3hVykCmtvd7UUm5J007+5NFuiwj0bMroJa293/VcF/w8d1l5Ka2XSJcq/uUgtEhXuCyUsOKMEai3LGw+cOBd4bFh7qV+67soIvZF2UunG4yj/5iK1ULgHCLueGeU6Zy3LGxtZCvnuX/1NhN5IO6m0K6LWu0uzKdybLGx01uxyMbqLNV2i3LgmUguFe5OFjc60l7VU5lrrLk2lcA/QyJy7SD0mHQZfPqKAl6ZRuAdoZM5dpF7F2vAizaB67iJtZKLKDWzDo9l5W/lpkw8JonAXSYjiBiDFOvHZidy8TUJEijQtI9Jm1m99PXDuPWgDENWlkTAKd5E2UxyRlwd8WP0Z1aWRIJHC3cxuNbNjZnbczIZCjvmymb1rZkfN7L80t5sinSVoRK5aRFKLquFuZt3AM8BtwBrgXjNbU3bMKmAzsN7dPw1sakFfRTpK+Yh8cMNqMmX762Z6uhncsHohuyUJEeWC6o3AcXc/AWBmLwF3AO+WHPNV4Bl3Pwfg7h82u6MinaZ8RF68aKrVMhJFlHDvA0p38R0Hbio75ucAzGw/0A087u7/rSk9TLErlvQElhHQTkwCcMsnl7J+6+vzglxhLlFECfegGzPL7+dZBKwCbgaWA//dzH7e3SfmvJHZA8ADACtWrKi5s2nzk7KVD9XapbO8eiirZY9StygXVMeBa0oeLwdOBxzzx+6ed/e/BI5RCPs53P1Zdx9w94GlS5fW2+fUuBBScCasXTqLlj1KI6KE+9vAKjNbaWaLgXuA3WXHDAO3AJjZ1RSmaU40s6MiomWPEl3VcHf3j4GHgH3Ae8BOdz9qZk+a2e3Th+0DfmRm7wJvAIPu/qNWdVqkU2nZo0QVqfyAu+8F9pa1PVbydwcenv4jIi2gZY9SC92hKpIQlzR7xxdJNf20iCTEuQt5Ht55WDXfJRKFu0iCTDls2nGYlUN7QguMiYDCXSSRnPACYyKgcA+kbfYkKbT2XcIo3ANomz1JEq19lyDaiSnApYu7OX9xfgmASxd3BxwtEq/yte/aik9A4R7oQkCwV2oXiUsXzFn7rq34pEjTMgEWalrGQibxw9pFypVWIRoezfLIziOqSSNAwkbuPV0QVFMrqfd2dAFBvwsk9NuRmGzeNcbIB2d59VCWSQ8egmhevvMkKtwnQ4bOYe3tLm3fj8Qjl59k+8FTocEOqknTiRI1SJwK+dkNaxfpFJWCXTVpOlOiwl1Eavf0nWt1MbUDKdxFUk7B3pkU7iIp1qe59o6lcBdJKc21d7ZEhXtvpqemdpFOprn2zpaocP/0sstqahfpVH29GQV7h0tUuB84ca6mdpFO1X+V5to7XaLCPWwtb6U1viKdaP/7Z1XnvcMlKtxFJLrNu96JuwsSI4W7SErlggoxScdIVLiriqKISDSJCvewqXVNuYsE6x/aw6f+9Xc1/96BEhXuIlK7XH6KTTsO0z+0h/VbX1fQdwiFu0gHKe7MpIBPP4W7SIfJ5Sd5ZOcRBXzKKdxFOtCku0bwKadwF+lQ2ls13RTuIh1Me6umV6RwN7NbzeyYmR03s6EKx/2KmbmZDTSviyLSKtpbNb2qhruZdQPPALcBa4B7zWxNwHGXAf8cONjsTopIa5yeyGmJZEpFGbnfCBx39xPufhF4Cbgj4Lh/C/w28OMm9k9EWqh4/192IsemHYe577m3Yu2PNE+UcO8DTpU8Hp9um2Fm64Br3P2/NrFvIrLA9r9/li3DY3F3Q5ogSrgHVW6ZueHfzLqArwOPVH0jswfMbMTMRs6cORO9lyKyYLYfPFX9IGl7iyIcMw5cU/J4OXC65PFlwM8Db1qhgtffAXab2e3uPlL6Ru7+LPAswMDAgCrCiLShSXeGR7Ns23eM0xM5lvVmGNywWjs7JUyUcH8bWGVmK4EscA/wleKT7v4RcHXxsZm9CfyL8mAXkWToMti8a4xcfhKYLVkAKOATpOq0jLt/DDwE7APeA3a6+1Eze9LMbm91B0VkYU05M8FepBuekifKyB133wvsLWt7LOTYmxvvloi0m+xEjpVDeypO02g6p31ECncRESispAibphkezWo6p42o/ICI1CxommbbvmOazmkjCncRqUt5XZqwOjWqXxMPhbuI1KW8Lk1YnRrVr4mHwl1Eapbp6WZww+o5bYMbVpPp6a56nCwMXVAVkZpcsaSHf/OPPj3vImnxsVbLtAeFu4hE0hchrDeu61OYtwmFu4hEsn/os3F3QWqgOXcRieQL/+HNuLsgNVC4i0gkP/jwPDf91vfi7oZEpHAXkcj+z99c1M5NCaFwF5GaZSdyfG3HYW3s0cYU7iJSFwdePHBSI/g2pXAXkbo5qHZMm9JSSBFpSHntmOHRLI9+Z4zzFwtFxAy47xdX8NTGtTH0rnMp3EWkIaW1Y4ZHszzy8hEmp2Z30XTghQMnARTwC0jTMiLSkP6rZsN9275jc4K9lDbeXlgKdxFpyIET52b+Xqm876QHh760hsJdRBpSGtqXZ3pCj+uyheiNFCncRaRhn9i8hy3DY/zfH+dDj5lytGxyASncRaRhU164aBoy3T5DyyYXjsJdRBZMdiJH/9Aern/iTzSKbzGFu4gsuIlcnod3HFbAt5DCXURiMQU8vvto3N1ILYW7iMRmIhd+AVYao3AXEUkhlR8Qkdhcurib9Vtfr3lD7eHRrDbirkLhLiKxOX9xkvMXC3e1ZidybN5VqA9fKaiHR7Ns3jVGLj9Z0+s6jaZlRKRt5PKTVdfCb9t3bCbYa3ldp4kU7mZ2q5kdM7PjZjYU8PzDZvaumb1jZn9qZtc2v6si0gkq1aep9Hy113WaquFuZt3AM8BtwBrgXjNbU3bYKDDg7p8BXgF+u9kdFZHOUFpCuJbnq72u00QZud8IHHf3E+5+EXgJuKP0AHd/w90vTD88ACxvbjdFpBNkeroZ3LC64jGDG1aT6emu+XWdJsoF1T6gtBDzOHBThePvB77bSKdEpPN0Gdx1Q1/Vi6LF57VaprIo4R5UqDOwPJCZ/SowAPz9kOcfAB4AWLFiRcQuikgnmHJ49VCWgWuvjBTwCvPKokzLjAPXlDxeDpwuP8jMPg88Ctzu7j8JeiN3f9bdB9x9YOnSpfX0V0RSrNKql+HRLOu3vs7KoT2s3/q66tJUEWXk/jawysxWAlngHuArpQeY2Trg94Bb3f3DpvdSRDpGdiLHyqE9c6Zbmrm2vVNugKoa7u7+sZk9BOwDuoFvuftRM3sSGHH33cA24G8BL5sZwEl3v72F/RaRFHMKAb5px2FGPjjLG98/E7q2vZZg7qQboCLdoerue4G9ZW2Plfz9803ul4gIUNgEJEyta9sr3QCVtnDXHaoikli1rm3vpBugFO4ikkhGYVqllournXQDlMJdRBLHmF2PXZw3jxLwnXQDlMJdRBKn/EabqIXDNq7r4+k719LXm8GAvt4MT9+5NnXz7aCSvyKSEtmI8+adcgOURu4ikhq6sWmWwl1EUmPwlSMK+GkKdxFJjfyk88RrR+PuRltQuItIqpy7kI+7C21BF1RFJHX6h/ZwxZIe3OGjXD7VNWTCaOQuIql07kKeiVx+pk7N13YcZsvwWNzdWjAauYtIR3AKdWpeOHCSvg4YyWvkLiIdp5a7WpNK4S4iHSnqXa1JpXAXkY4V9a7WJFK4i0jHMtJ7V6vCXUQ6lkNqp2YU7iLS0Yp7tqZt020thRSRjle6Fn7kg7M8tXHtnOeTuKm2wl1EZJoDL07v2frG989weiJH75Ie/t+PPyY/Vagin5RNtTUtIyJSonizU3Yih1O407UY7EVJWEapcBcRqUO7b6qtcBcRqUO7b6qtOXcRkRqFbardThdeEzVy/8bd19fUHoefuWxx5PZVP31p4LFh7bUeIyLNF7ap9vBols27xmbm6uOuX5OocN+4ro9v3H39nJ3Lv3H39U3/ZFx/3ZU1tZc6+OgX5gX5z1y2mIOPfmHesd97+OZ5Ib3qpy/lew/fXLUfYa/ta/NfFUWSrNuM/UOfDcycbfuOkctPzmmL88KruXv1o1pgYGDAR0ZGYvnaUdz33Fvsf//szOP1113Ji1/9e23fj+LoofSHLNPTzV039M0s7VrWm+GWTy7lheklX6WMwmqB0r8bYAZTAT8q3WZMus95nUia/XDrLwe2rxzaE/j/gAF/GfKaepjZIXcfqHac5txDxBHkQWrtR3FEEWXeb+DaKyPPDzYyl1h8bXYiN/Nh0NebYcniLn7w4fmavj+ROFX6zXhZbyawEFlcF141cpfE2zI8xosHTuo3B2mKvt4M/Vdl5vzGXNTTBdu+FDwVHPRzWPyNtpmbg0QduSvcpSMNj2b5V7ve4UJ+CkDTSlIXM3CHJT1dMz9LlSzp6eKnerqZuFD/vq5NnZYxs1uB/wh0A7/v7lvLnv8p4NvADcCPgLvd/Yc19Tii/qE989rC5sDi+jq1vLbasZWeD3suqF0qU7BLPYpj4yjBXjyueGyryxhUHbmbWTfwv4EvAOPA28C97v5uyTG/DnzG3R80s3uAf+zud1d633pG7pVCq5kB38jXqeW11Y5VSIukX19vhv1Dn418fNSRe5SlkDcCx939hLtfBF4C7ig75g7gD6f//grwOTOzyL0VEelQrSpjECXc+4BTJY/Hp9sCj3H3j4GPgKvK38jMHjCzETMbOXPmTH09FhFJkVatpokS7kEj8PK5nCjH4O7PuvuAuw8sXbo0Sv9ERFIrrIxBM0QJ93HgmpLHy4HTYceY2SLgcmD+OiIRkRZb3B08I9zVwERx8aXl71GcfL5iSU+kMO3N9HDFkp6ZO+yDyhg0S5QLqosoXFD9HJClcEH1K+5+tOSY3wDWllxQvdPdv1zpfetdCqnVMlotI+lWvjZ85IOzbD94ismArApbP95oAa92KgBWrqnr3M3si8A3KCyF/Ja7/5aZPQmMuPtuM7sE+CNgHYUR+z3ufqLSe2qdu4hI7Zq6zt3d9wJ7y9oeK/n7j4Ev1dpJERFpjURVhRQRkWgU7iIiKaRwFxFJIYW7iEgKKdxFRFJI4S4ikkIKdxGRFIptsw4zOwN8QKFUwUdlT0dpuxr465Z1sHJfWvHaasdWej7sOZ3baMfq3Nb/Wp3b1r027Nhr3b16cS53j/UP8Gw9bRTujo2lf614bbVjKz0f9pzOrc6tzm16z221P+0wLfNaA20LoZGvW8trqx1b6fmw53Ruox2rc1v/a3VuW/fahr6/2KZlGmVmIx6hvoLUTue2dXRuW0fndq52GLnX69m4O5BiOreto3PbOjq3JRI7chcRkXBJHrmLiEgIhbuISAop3EVEUigV4W5ml5rZH5rZc2Z2X9z9SRMz+4SZ/YGZvRJ3X9LGzDZO/8z+sZn9g7j7kyZm9ikz+10ze8XM/lnc/YlD24a7mX3LzD40s78oa7/VzI6Z2XEzG5puvhN4xd2/Cty+4J1NmFrOrbufcPf74+lp8tR4boenf2Z/Dbg7hu4mSo3n9j13fxD4MtCRyyPbNtyB54FbSxvMrBt4BrgNWAPca2ZrgOXAqenDJhewj0n1PNHPrdTmeWo/t1umn5fKnqeGc2tmtwP/A/jThe1me2jbcHf3P6Ow2XapG4Hj06PJi8BLwB3AOIWAhzb+ntpFjedWalDLubWCfwd8193/10L3NWlq/bl1993u/ktAR07VJi0I+5gdoUMh1PuAXcBdZvY7xHdLctIFnlszu8rMfhdYZ2ab4+la4oX93P4m8HngV8zswTg6lgJhP7c3m9k3zez3gL3xdC1ei+LuQI0soM3d/TzwTxe6MykTdm5/BCh4GhN2br8JfHOhO5MyYef2TeDNhe1Ke0nayH0cuKbk8XLgdEx9SRud29bRuW0dndsQSQv3t4FVZrbSzBYD9wC7Y+5TWujcto7Obevo3IZo23A3s+3AW8BqMxs3s/vd/WPgIWAf8B6w092PxtnPJNK5bR2d29bRua2NCoeJiKRQ247cRUSkfgp3EZEUUriLiKSQwl1EJIUU7iIiKaRwFxFJIYW7iEgKKdxFRFJI4S4ikkL/H/LjBddAaJEaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(len_diff, f1)\n",
    "plt.gca().set_xscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 1.        , -0.27873493],\n",
       "        [-0.27873493,  1.        ]]), array([[ 1.        , -0.24160674],\n",
       "        [-0.24160674,  1.        ]]))"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.corrcoef(len_diff, f1), np.corrcoef(np.log1p(len_diff), f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1840835784076519"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(0.25675646704517774+0.11654683276540687+0.17894743541237113)/3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Уникальные символы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2007738/2007738 [00:33<00:00, 59274.35it/s]\n"
     ]
    }
   ],
   "source": [
    "symbols = ''.join({c for text in tqdm(chain(first_sentences, titles), total=len(titles) + len(first_sentences)) for c in text})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\t',\n",
       " '\\n',\n",
       " ' ',\n",
       " '!',\n",
       " '\"',\n",
       " '#',\n",
       " '$',\n",
       " '%',\n",
       " '&',\n",
       " \"'\",\n",
       " '(',\n",
       " ')',\n",
       " '*',\n",
       " '+',\n",
       " ',',\n",
       " '-',\n",
       " '.',\n",
       " '/',\n",
       " ':',\n",
       " ';',\n",
       " '<',\n",
       " '=',\n",
       " '>',\n",
       " '?',\n",
       " '@',\n",
       " '[',\n",
       " '\\\\',\n",
       " ']',\n",
       " '^',\n",
       " '_',\n",
       " '`',\n",
       " '{',\n",
       " '|',\n",
       " '}',\n",
       " '\\x96',\n",
       " '\\x98',\n",
       " '\\xa0',\n",
       " '¡',\n",
       " '£',\n",
       " '¤',\n",
       " '©',\n",
       " '«',\n",
       " '¬',\n",
       " '\\xad',\n",
       " '®',\n",
       " '°',\n",
       " '¸',\n",
       " '»',\n",
       " '×',\n",
       " '˚',\n",
       " '́',\n",
       " '̆',\n",
       " '̇',\n",
       " '\\u200b',\n",
       " '\\u200e',\n",
       " '\\u200f',\n",
       " '‐',\n",
       " '‑',\n",
       " '–',\n",
       " '—',\n",
       " '―',\n",
       " '‘',\n",
       " '’',\n",
       " '“',\n",
       " '”',\n",
       " '„',\n",
       " '•',\n",
       " '…',\n",
       " '\\u202a',\n",
       " '′',\n",
       " '€',\n",
       " '№',\n",
       " '™',\n",
       " '−',\n",
       " '─',\n",
       " '╗',\n",
       " '\\ufeff',\n",
       " '＄',\n",
       " '�']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(_ for _ in symbols if not _.isalnum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2007738/2007738 [03:14<00:00, 10317.05it/s]\n"
     ]
    }
   ],
   "source": [
    "symb_freq = Counter()\n",
    "for text in tqdm(chain(first_sentences, titles), total=len(titles) + len(first_sentences)):\n",
    "    for c in text:\n",
    "        symb_freq[c] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(' ', 32005918),\n",
       " ('о', 23939338),\n",
       " ('и', 18469403),\n",
       " ('а', 18237326),\n",
       " ('е', 18017618),\n",
       " ('н', 14467269),\n",
       " ('р', 14095822),\n",
       " ('с', 13896311),\n",
       " ('т', 13307079),\n",
       " ('в', 11546841),\n",
       " ('л', 9067785),\n",
       " ('к', 7945272),\n",
       " ('п', 6791019),\n",
       " ('д', 6727731),\n",
       " ('м', 6322082),\n",
       " ('у', 5551596),\n",
       " ('г', 4249210),\n",
       " ('б', 4000556),\n",
       " ('я', 3549545),\n",
       " ('ы', 3439236),\n",
       " ('з', 3275300),\n",
       " ('й', 3116615),\n",
       " ('ь', 2704446),\n",
       " ('ч', 2208296),\n",
       " (',', 2193126),\n",
       " ('\\xa0', 1725141),\n",
       " ('х', 1722640),\n",
       " ('ж', 1612129),\n",
       " ('ц', 1536346),\n",
       " ('\"', 1281273),\n",
       " ('ф', 1197464),\n",
       " ('ш', 1196912),\n",
       " ('ю', 1111828),\n",
       " ('щ', 1064573),\n",
       " ('-', 841800),\n",
       " ('1', 517336),\n",
       " ('0', 506249),\n",
       " ('э', 494420),\n",
       " ('2', 435471),\n",
       " ('e', 224614),\n",
       " ('a', 220043),\n",
       " ('i', 204440),\n",
       " ('o', 179629),\n",
       " ('(', 178504),\n",
       " (')', 174979),\n",
       " ('5', 170927),\n",
       " ('3', 167642),\n",
       " ('r', 162101),\n",
       " ('s', 156804),\n",
       " ('n', 152618),\n",
       " ('t', 148334),\n",
       " ('4', 141145),\n",
       " (':', 129687),\n",
       " ('l', 107963),\n",
       " ('6', 98111),\n",
       " ('.', 97371),\n",
       " ('7', 94994),\n",
       " ('8', 94320),\n",
       " ('c', 93190),\n",
       " ('9', 90524),\n",
       " ('ъ', 82207),\n",
       " ('u', 74722),\n",
       " ('p', 71777),\n",
       " ('d', 69592),\n",
       " ('m', 68665),\n",
       " ('%', 62986),\n",
       " ('g', 58071),\n",
       " ('b', 51376),\n",
       " ('h', 49632),\n",
       " ('w', 46455),\n",
       " ('f', 40796),\n",
       " ('y', 40769),\n",
       " ('—', 38444),\n",
       " ('k', 37601),\n",
       " ('v', 24597),\n",
       " ('\\n', 22364),\n",
       " ('z', 20454),\n",
       " (';', 17772),\n",
       " ('x', 16720),\n",
       " ('j', 14874),\n",
       " ('–', 12984),\n",
       " ('/', 11762),\n",
       " ('$', 8325),\n",
       " ('«', 7494),\n",
       " ('ё', 7270),\n",
       " ('?', 7252),\n",
       " ('»', 7192),\n",
       " ('!', 7104),\n",
       " ('q', 6813),\n",
       " ('&', 6641),\n",
       " (\"'\", 5676),\n",
       " ('№', 4379),\n",
       " ('ł', 3878),\n",
       " ('*', 2881),\n",
       " ('>', 2664),\n",
       " ('ę', 2562),\n",
       " ('ó', 2040),\n",
       " ('ą', 1652),\n",
       " ('+', 1539),\n",
       " ('ż', 1331)]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "symb_freq.most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' ,\\xa0\"-():.%—\\n;–/$«?»!&\\'№*>+'"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join([_[0] for _ in symb_freq.most_common(200) if (not _[0].isalnum()) and (_[1] > 1000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "',\"-():.%—\\n;–/$«?»!&\\'№*>+][…‑“”'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string\n",
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "regex_punctuation_replace = re.compile('\\s([!%,.:;?])')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'авроы! выапоц, pdsof'"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex_punctuation_replace.sub(r'\\1', \"авроы ! выапоц , pdsof\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'для продолжения силовой операции в\\xa0донбассе необходимо пополнение резервов военнослужащих, заявил в\\xa0воскресенье в\\xa0харькове министр обороны степан полторак, передает \"радио свобода\"'"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_sentences[np.random.choice(len(first_sentences))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}