main-github.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd \n",
    "import unicodedata\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras as krs\n",
    "from tensorflow.keras.layers import Embedding\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.losses import SparseCategoricalCrossentropy, CategoricalCrossentropy\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer \n",
    "from tensorflow.keras.optimizers.schedules import LearningRateSchedule\n",
    "import csv\n",
    "\n",
    "BUFFER_SIZE = 20000\n",
    "BATCH_SIZE = 64\n",
    "embedding_dim = 50 # first it was 200"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading the data and droppping useless columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>short</th>\n",
       "      <th>long</th>\n",
       "      <th>Source</th>\n",
       "      <th>Time</th>\n",
       "      <th>Publish Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4 ex-bank officials booked for cheating bank o...</td>\n",
       "      <td>The CBI on Saturday booked four former officia...</td>\n",
       "      <td>The New Indian Express</td>\n",
       "      <td>09:25:00</td>\n",
       "      <td>2017-03-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Supreme Court to go paperless in 6 months: CJI</td>\n",
       "      <td>Chief Justice JS Khehar has said the Supreme C...</td>\n",
       "      <td>Outlook</td>\n",
       "      <td>22:18:00</td>\n",
       "      <td>2017-03-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>At least 3 killed, 30 injured in blast in Sylh...</td>\n",
       "      <td>At least three people were killed, including a...</td>\n",
       "      <td>Hindustan Times</td>\n",
       "      <td>23:39:00</td>\n",
       "      <td>2017-03-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Why has Reliance been barred from trading in f...</td>\n",
       "      <td>Mukesh Ambani-led Reliance Industries (RIL) wa...</td>\n",
       "      <td>Livemint</td>\n",
       "      <td>23:08:00</td>\n",
       "      <td>2017-03-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Was stopped from entering my own studio at Tim...</td>\n",
       "      <td>TV news anchor Arnab Goswami has said he was t...</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>23:24:00</td>\n",
       "      <td>2017-03-25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               short  \\\n",
       "0  4 ex-bank officials booked for cheating bank o...   \n",
       "1     Supreme Court to go paperless in 6 months: CJI   \n",
       "2  At least 3 killed, 30 injured in blast in Sylh...   \n",
       "3  Why has Reliance been barred from trading in f...   \n",
       "4  Was stopped from entering my own studio at Tim...   \n",
       "\n",
       "                                                long                 Source   \\\n",
       "0  The CBI on Saturday booked four former officia...  The New Indian Express   \n",
       "1  Chief Justice JS Khehar has said the Supreme C...                 Outlook   \n",
       "2  At least three people were killed, including a...         Hindustan Times   \n",
       "3  Mukesh Ambani-led Reliance Industries (RIL) wa...                Livemint   \n",
       "4  TV news anchor Arnab Goswami has said he was t...                 YouTube   \n",
       "\n",
       "      Time  Publish Date  \n",
       "0  09:25:00   2017-03-26  \n",
       "1  22:18:00   2017-03-25  \n",
       "2  23:39:00   2017-03-25  \n",
       "3  23:08:00   2017-03-25  \n",
       "4  23:24:00   2017-03-25  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_unprocessed_news = pd.read_excel('data.xlsx')\n",
    "data_unprocessed_news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>short</th>\n",
       "      <th>long</th>\n",
       "      <th>Source</th>\n",
       "      <th>Time</th>\n",
       "      <th>Publish Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1933</th>\n",
       "      <td>Uber India to check drivers&amp;#39; ID in real ti...</td>\n",
       "      <td>Uber has launched a real-time ID check for dri...</td>\n",
       "      <td>YourStory</td>\n",
       "      <td>16:46:00</td>\n",
       "      <td>2017-03-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>720</th>\n",
       "      <td>Tom Cruise training for Mission: Impossible 6 ...</td>\n",
       "      <td>Actor Tom Cruise has been training for a stunt...</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>19:03:00</td>\n",
       "      <td>2017-03-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48970</th>\n",
       "      <td>&amp;#39;Easter 2016&amp;#39; top Google trend today</td>\n",
       "      <td>On the day the festival Easter was celebrated ...</td>\n",
       "      <td>Google</td>\n",
       "      <td>23:48:00</td>\n",
       "      <td>2016-03-27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51836</th>\n",
       "      <td>Women raped, harassed at NH1 near Delhi: Report</td>\n",
       "      <td>According to a report by The Tribune, women co...</td>\n",
       "      <td>India Today</td>\n",
       "      <td>21:24:00</td>\n",
       "      <td>2016-02-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29389</th>\n",
       "      <td>Whatever wicket we get, we will adapt and play...</td>\n",
       "      <td>Indian cricket team coach Anil Kumble has said...</td>\n",
       "      <td>Photo Gallery</td>\n",
       "      <td>17:24:00</td>\n",
       "      <td>2016-09-21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   short  \\\n",
       "1933   Uber India to check drivers&#39; ID in real ti...   \n",
       "720    Tom Cruise training for Mission: Impossible 6 ...   \n",
       "48970       &#39;Easter 2016&#39; top Google trend today   \n",
       "51836    Women raped, harassed at NH1 near Delhi: Report   \n",
       "29389  Whatever wicket we get, we will adapt and play...   \n",
       "\n",
       "                                                    long        Source   \\\n",
       "1933   Uber has launched a real-time ID check for dri...      YourStory   \n",
       "720    Actor Tom Cruise has been training for a stunt...        YouTube   \n",
       "48970  On the day the festival Easter was celebrated ...         Google   \n",
       "51836  According to a report by The Tribune, women co...    India Today   \n",
       "29389  Indian cricket team coach Anil Kumble has said...  Photo Gallery   \n",
       "\n",
       "          Time  Publish Date  \n",
       "1933   16:46:00   2017-03-13  \n",
       "720    19:03:00   2017-03-21  \n",
       "48970  23:48:00   2016-03-27  \n",
       "51836  21:24:00   2016-02-24  \n",
       "29389  17:24:00   2016-09-21  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.utils import shuffle\n",
    "# shuffling the data \n",
    "data_unprocessed_news = shuffle(data_unprocessed_news)\n",
    "data_unprocessed_news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((55104, 1), (55104, 1))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summaries, longreview = pd.DataFrame(), pd.DataFrame()\n",
    "summaries['short'] = data_unprocessed_news['short']#[:data_to_use]\n",
    "longreview['long'] = data_unprocessed_news['long']#[:data_to_use]\n",
    "(summaries.shape,longreview.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleaning the data for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replacing many abbreviations and lower casing the words\n",
    "def clean_words(sentence):\n",
    "    sentence = str(sentence).lower()\n",
    "    sentence = unicodedata.normalize('NFKD', sentence).encode('ascii', 'ignore').decode('utf-8', 'ignore') # for converting é to e and other accented chars\n",
    "    sentence = re.sub(r\"http\\S+\",\"\",sentence)\n",
    "    sentence = re.sub(r\"there's\", \"there is\", sentence)\n",
    "    sentence = re.sub(r\"i'm\", \"i am\", sentence)\n",
    "    sentence = re.sub(r\"he's\", \"he is\", sentence)\n",
    "    sentence = re.sub(r\"she's\", \"she is\", sentence)\n",
    "    sentence = re.sub(r\"it's\", \"it is\", sentence)\n",
    "    sentence = re.sub(r\"that's\", \"that is\", sentence)\n",
    "    sentence = re.sub(r\"what's\", \"that is\", sentence)\n",
    "    sentence = re.sub(r\"where's\", \"where is\", sentence)\n",
    "    sentence = re.sub(r\"how's\", \"how is\", sentence)\n",
    "    sentence = re.sub(r\"\\'ll\", \" will\", sentence)\n",
    "    sentence = re.sub(r\"\\'ve\", \" have\", sentence)\n",
    "    sentence = re.sub(r\"\\'re\", \" are\", sentence)\n",
    "    sentence = re.sub(r\"\\'d\", \" would\", sentence)\n",
    "    sentence = re.sub(r\"\\'re\", \" are\", sentence)\n",
    "    sentence = re.sub(r\"won't\", \"will not\", sentence)\n",
    "    sentence = re.sub(r\"can't\", \"cannot\", sentence)\n",
    "    sentence = re.sub(r\"n't\", \" not\", sentence)\n",
    "    sentence = re.sub(r\"n'\", \"ng\", sentence)\n",
    "    sentence = re.sub(r\"'bout\", \"about\", sentence)\n",
    "    sentence = re.sub(r\"'til\", \"until\", sentence)\n",
    "    sentence = re.sub(r\"\\\"\", \"\", sentence)\n",
    "    sentence = re.sub(r\"\\'\", \"\", sentence)\n",
    "    sentence = re.sub(r' s ', \"\",sentence)\n",
    "    sentence = re.sub(r\"&39\", \"\", sentence) # the inshorts data has this in it\n",
    "    sentence = re.sub(r\"&34\", \"\", sentence) # the inshorts data has this in it\n",
    "    sentence = re.sub(r\"[\\[\\]\\\\0-9()\\\"$#%/@;:<>{}`+=~|.!?,-]\", \"\", sentence)\n",
    "    sentence = re.sub(r\"&\", \"\", sentence)\n",
    "    sentence = re.sub(r\"\\\\n\", \"\", sentence)\n",
    "    sentence = sentence.strip()\n",
    "    return sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "summaries['short'] = summaries['short'].apply(lambda x: clean_words(x))\n",
    "longreview['long'] = longreview['long'].apply(lambda x: clean_words(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>long</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1933</th>\n",
       "      <td>uber has launched a realtime id check for driv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>720</th>\n",
       "      <td>actor tom cruise has been training for a stunt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48970</th>\n",
       "      <td>on the day the festival easter was celebrated ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51836</th>\n",
       "      <td>according to a report by the tribune women com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29389</th>\n",
       "      <td>indian cricket team coach anil kumble has said...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    long\n",
       "1933   uber has launched a realtime id check for driv...\n",
       "720    actor tom cruise has been training for a stunt...\n",
       "48970  on the day the festival easter was celebrated ...\n",
       "51836  according to a report by the tribune women com...\n",
       "29389  indian cricket team coach anil kumble has said..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "longreview.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>short</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1933</th>\n",
       "      <td>&lt;startseq&gt; uber india to check drivers id in r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>720</th>\n",
       "      <td>&lt;startseq&gt; tom cruise training for mission imp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48970</th>\n",
       "      <td>&lt;startseq&gt; easter  top google trend today &lt;end...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51836</th>\n",
       "      <td>&lt;startseq&gt; women raped harassed at nh near del...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29389</th>\n",
       "      <td>&lt;startseq&gt; whatever wicket we get we will adap...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   short\n",
       "1933   <startseq> uber india to check drivers id in r...\n",
       "720    <startseq> tom cruise training for mission imp...\n",
       "48970  <startseq> easter  top google trend today <end...\n",
       "51836  <startseq> women raped harassed at nh near del...\n",
       "29389  <startseq> whatever wicket we get we will adap..."
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# adding start and end token to the senteces of label \n",
    "start_token, end_token = '<startseq>' , '<endseq>'\n",
    "summaries = summaries.apply(lambda x: start_token + ' ' + x + ' ' + end_token)\n",
    "summaries.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5510, 49594)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_split = 0.1\n",
    "# train validation split\n",
    "summaries_train = summaries[int(len(summaries)*val_split):]\n",
    "summaries_val = summaries[:int(len(summaries)*val_split)]\n",
    "longreview_train = longreview[int(len(summaries)*val_split):]\n",
    "longreview_val = longreview[:int(len(summaries)*val_split)]\n",
    "\n",
    "len(longreview_val),len(longreview_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(long    nasas curiosity rover today shared a series of...\n",
       " Name: 52712, dtype: object,\n",
       " short    <startseq> nasas curiosity rover shares valent...\n",
       " Name: 52712, dtype: object)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "longreview_train.iloc[0], summaries_train.iloc[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "finding the maximum length of questions and answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "percentile 90 of length of news: 60.0\n",
      "longest sentence:  66\n",
      "\n",
      "percentile 90 of length of summaries: 12.0\n",
      "longest sentence:  16\n",
      "\n",
      "max-length longreview chosen for training:  60\n",
      "max-length summaries chosen for training:  12\n"
     ]
    }
   ],
   "source": [
    "# because there are senteces with unusually long lengths, \n",
    "# we caculate the max length that 95% of sentences are shorter than that\n",
    "def max_length(shorts, longs, prct):\n",
    "    # Create a list of all the captions\n",
    "    \n",
    "    length_longs = list(len(d.split()) for d in longs)\n",
    "    length_shorts = list(len(d.split()) for d in shorts)\n",
    "\n",
    "    print('percentile {} of length of news: {}'.format(prct,np.percentile(length_longs, prct)))\n",
    "    print('longest sentence: ', max(length_longs))\n",
    "    print()\n",
    "    print('percentile {} of length of summaries: {}'.format(prct,np.percentile(length_shorts, prct)))\n",
    "    print('longest sentence: ', max(length_shorts))\n",
    "    print()\n",
    "    return int(np.percentile(length_longs, prct)),int(np.percentile(length_shorts, prct))\n",
    "\n",
    "# selecting sentence length based on the percentile of data that fits in the length\n",
    "max_len_news, max_len_summary= max_length(summaries_train['short'].to_list(), longreview_train['long'].to_list(), 90)\n",
    "\n",
    "\n",
    "print('max-length longreview chosen for training: ', max_len_news)\n",
    "print('max-length summaries chosen for training: ', max_len_summary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset prepration\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# making a vocabulary of the words \n",
    "def create_vocab(shorts, longs = None, minimum_repeat = 3):\n",
    "\n",
    "    # Create a list of all the captions\n",
    "    all_captions = []\n",
    "    for s in shorts:\n",
    "        all_captions.append(s)\n",
    "\n",
    "    # Consider only words which occur at least minimum_occurrence times in the corpus\n",
    "    word_counts = {}\n",
    "    nsents = 0\n",
    "    for sent in all_captions:\n",
    "        nsents += 1\n",
    "        for w in sent.split(' '):\n",
    "            word_counts[w] = word_counts.get(w, 0) + 1\n",
    "\n",
    "    vocab = [w for w in word_counts if word_counts[w] >= minimum_repeat]\n",
    "    \n",
    "    vocab = list(set(vocab))\n",
    "    return vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<endseq>',\n",
       " '<startseq>',\n",
       " 'a',\n",
       " 'aa',\n",
       " 'aadhaar',\n",
       " 'aadhaarbased',\n",
       " 'aadmi',\n",
       " 'aam',\n",
       " 'aamir',\n",
       " 'aamirs']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# each word in the vocabulary must be used in the data atleast minimum_repeat times\n",
    "vocab_dec = create_vocab(summaries_train['short'].to_list(), minimum_repeat=5) # here we just use the words in vocabulary of summaries\n",
    "# removing one character words from vocab except for 'a'\n",
    "for v in vocab_dec:\n",
    "    if len(v) == 1 and v!='a' and v!='i':\n",
    "        vocab_dec.remove(v) \n",
    "        \n",
    "vocab_dec = sorted(vocab_dec)[1:] # [1:] is for the '' \n",
    "vocab_dec[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['****ing',\n",
       " 'a',\n",
       " 'aa',\n",
       " 'aaa',\n",
       " 'aadat',\n",
       " 'aadhaar',\n",
       " 'aadhaarbased',\n",
       " 'aadhaarenabled',\n",
       " 'aadhaarlinked',\n",
       " 'aadhar']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# each word in the vocabulary must be used in the data atleast minimum_repeat times\n",
    "vocab_enc = create_vocab(longreview_train['long'].to_list(), minimum_repeat=3) # here we just use the words in vocabulary of summaries\n",
    "# removing one character words from vocab except for 'a'\n",
    "for v in vocab_enc:\n",
    "    if len(v) == 1 and v!='a' and v!='i':\n",
    "        vocab_enc.remove(v) \n",
    "        \n",
    "vocab_enc = sorted(vocab_enc)[1:] # [1:] is for the '' \n",
    "vocab_enc[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32447, 8884)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oov_token = '<UNK>'\n",
    "filters = '!\"#$%&()*+,-./:;=?@[\\\\]^_`{|}~\\t\\n' # making sure all the last non digit non alphabet chars are removed\n",
    "document_tokenizer = krs.preprocessing.text.Tokenizer(filters = filters,oov_token=oov_token)\n",
    "summary_tokenizer = krs.preprocessing.text.Tokenizer(filters = filters,oov_token=oov_token)\n",
    "document_tokenizer.fit_on_texts(vocab_enc)\n",
    "summary_tokenizer.fit_on_texts(vocab_dec)#summaries_train['short'])\n",
    "\n",
    "# caculating number of words in vocabulary of encoder and decoder\n",
    "# they are important for positional encoding\n",
    "encoder_vocab_size = len(document_tokenizer.word_index) + 1 \n",
    "decoder_vocab_size = len(summary_tokenizer.word_index) + 1\n",
    "\n",
    "# vocab_size\n",
    "encoder_vocab_size, decoder_vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "ixtoword_enc = {} # index to word dic\n",
    "ixtoword_dec = {} # index to word dic\n",
    "\n",
    "wordtoix_enc = document_tokenizer.word_index # word to index dic\n",
    "ixtoword_enc[0] = '<PAD0>' # no word in vocab has index 0. but padding is indicated with 0\n",
    "ixtoword_dec[0] = '<PAD0>' # no word in vocab has index 0. but padding is indicated with 0\n",
    "\n",
    "for w in document_tokenizer.word_index:\n",
    "    ixtoword_enc[document_tokenizer.word_index[w]] = w\n",
    "################################################\n",
    "wordtoix_dec = summary_tokenizer.word_index # word to index dic\n",
    "\n",
    "for w in summary_tokenizer.word_index:\n",
    "    ixtoword_dec[summary_tokenizer.word_index[w]] = w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# assign a number to each word inorder to find it in word embeddings\n",
    "inputs = document_tokenizer.texts_to_sequences(longreview_train['long'])\n",
    "targets = summary_tokenizer.texts_to_sequences(summaries_train['short'])\n",
    "inputs_val = document_tokenizer.texts_to_sequences(longreview_val['long'])\n",
    "targets_val = summary_tokenizer.texts_to_sequences(summaries_val['short'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = krs.preprocessing.sequence.pad_sequences(inputs, maxlen=max_len_news, padding='post', truncating='post')\n",
    "targets = krs.preprocessing.sequence.pad_sequences(targets, maxlen=max_len_summary, padding='post', truncating='post')\n",
    "inputs_val = krs.preprocessing.sequence.pad_sequences(inputs_val, maxlen=max_len_news, padding='post', truncating='post')\n",
    "targets_val = krs.preprocessing.sequence.pad_sequences(targets_val, maxlen=max_len_summary, padding='post', truncating='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# validate train split\n",
    "dataset = tf.data.Dataset.from_tensor_slices((inputs,targets)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
    "dataset_val = tf.data.Dataset.from_tensor_slices((inputs_val,targets_val)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE*2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "longreview_val.reset_index(inplace=True, drop=True)\n",
    "summaries_val.reset_index(inplace=True, drop=True)\n",
    "longreview_train.reset_index(inplace=True, drop=True)\n",
    "summaries_train.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def hist(history):\n",
    "    plt.title('Loss')\n",
    "\n",
    "    x= [i[0] for i in history['val']]\n",
    "    y=[i[1] for i in history['val']]\n",
    "    plt.plot(x,y,'x-')\n",
    "    \n",
    "    x= [i[0] for i in history['train']]\n",
    "    y=[i[1] for i in history['train']]    \n",
    "    plt.plot(x,y,'o-')\n",
    "\n",
    "    plt.legend(['validation','train'])\n",
    "    plt.show()\n",
    "    print('smallest val loss:', sorted(history['val'],key=lambda x: x[1])[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Scaled Dot Product\n",
    "![](files/z-score.png)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the job of this function is to calculate the above equation\n",
    "def scaled_dot_product_attention(q, k, v, mask):\n",
    "    matmul_qk = tf.matmul(q, k, transpose_b=True)\n",
    "\n",
    "    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
    "    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
    "\n",
    "    if mask is not None:\n",
    "        scaled_attention_logits += (mask * -1e9)  \n",
    "\n",
    "    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)\n",
    "\n",
    "    output = tf.matmul(attention_weights, v)\n",
    "    return output, attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### multi-headed attention\n",
    "![](files/multi-head.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(krs.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.num_heads = num_heads\n",
    "        self.d_model = d_model\n",
    "\n",
    "        assert d_model % self.num_heads == 0\n",
    "\n",
    "        self.depth = d_model // self.num_heads # The dimensions of Q, K, V are called depth\n",
    "\n",
    "        # the input of these 3 layers are the same: X\n",
    "        self.wq = krs.layers.Dense(d_model,kernel_regularizer=krs.regularizers.l2(l=lmbda_l2))\n",
    "        self.wk = krs.layers.Dense(d_model,kernel_regularizer=krs.regularizers.l2(l=lmbda_l2))\n",
    "        self.wv = krs.layers.Dense(d_model,kernel_regularizer=krs.regularizers.l2(l=lmbda_l2))\n",
    "\n",
    "        self.dense = tf.keras.layers.Dense(d_model,kernel_regularizer=krs.regularizers.l2(l=lmbda_l2))\n",
    "    \n",
    "    # reshape the Q,K,V \n",
    "    def split_heads(self, x, batch_size):\n",
    "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
    "        return tf.transpose(x, perm=[0, 2, 1, 3])\n",
    "    \n",
    "    def call(self, v, k, q, mask):\n",
    "        batch_size = tf.shape(q)[0]\n",
    "        \n",
    "        # learn the Q,K,V matrices (the layers' weightes)\n",
    "        q = self.wq(q)\n",
    "        k = self.wk(k)\n",
    "        v = self.wv(v)\n",
    "        \n",
    "        # reshape them\n",
    "        q = self.split_heads(q, batch_size)\n",
    "        k = self.split_heads(k, batch_size)\n",
    "        v = self.split_heads(v, batch_size)\n",
    "\n",
    "        scaled_attention, attention_weights = scaled_dot_product_attention(\n",
    "            q, k, v, mask)\n",
    "\n",
    "        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n",
    "        \n",
    "        # the last dens layer expect one vector so we use concat\n",
    "        concat_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model))\n",
    "        output = self.dense(concat_attention)\n",
    "            \n",
    "        return output, attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### positional encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_angles(position, i, d_model):\n",
    "    angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(d_model))\n",
    "    return position * angle_rates\n",
    "\n",
    "# The dimension of positional encodings is the same as\n",
    "# the embeddings (d_model) for facilitating the summation of both.\n",
    "def positional_encoding(position, d_model):\n",
    "    angle_rads = get_angles(\n",
    "        np.arange(position)[:, np.newaxis],\n",
    "        np.arange(d_model)[np.newaxis, :],\n",
    "        d_model\n",
    "    )\n",
    "\n",
    "    # apply sin to even indices in the array; 2i\n",
    "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
    "\n",
    "    # apply cos to odd indices in the array; 2i+1\n",
    "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
    "\n",
    "    pos_encoding = angle_rads[np.newaxis, ...]\n",
    "\n",
    "    return tf.cast(pos_encoding, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Embeddings preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading glove...\n",
      "GloVe  50  loaded!\n",
      "Loading glove...\n",
      "GloVe  50  loaded!\n"
     ]
    }
   ],
   "source": [
    " # Making the embedding mtrix\n",
    "def make_embedding_layer(vocab_len, wordtoix, embedding_dim=200, glove=True, glove_path= '../glove'):\n",
    "    if glove == False:\n",
    "        print('Just a zero matrix loaded')\n",
    "        embedding_matrix = np.zeros((vocab_len, embedding_dim)) # just a zero matrix \n",
    "    else:\n",
    "        print('Loading glove...')\n",
    "        glove_dir = glove_path\n",
    "        embeddings_index = {} \n",
    "        f = open(os.path.join(glove_dir, 'glove.6B.'+str(embedding_dim)+'d.txt'), encoding=\"utf-8\")\n",
    "        for line in f:\n",
    "            values = line.split()\n",
    "            word = values[0]\n",
    "            coefs = np.asarray(values[1:], dtype='float32')\n",
    "            embeddings_index[word] = coefs\n",
    "        f.close()\n",
    "        # Get n-dim dense vector for each of the vocab_rocc\n",
    "        embedding_matrix = np.zeros((vocab_len, embedding_dim)) # to import as weights for Keras Embedding layer\n",
    "        for word, i in wordtoix.items():\n",
    "            embedding_vector = embeddings_index.get(word)\n",
    "            if embedding_vector is not None:\n",
    "                # Words not found in the embedding index will be all zeros\n",
    "                embedding_matrix[i] = embedding_vector\n",
    "        \n",
    "        print(\"GloVe \",embedding_dim, ' loaded!')\n",
    "\n",
    "    embedding_layer = Embedding(vocab_len, embedding_dim, mask_zero=True, trainable=False) # we have a limited vocab so we \n",
    "                                                                                           # do not train the embedding layer\n",
    "                                                                                           # we use 0 as padding so => mask_zero=True\n",
    "    embedding_layer.build((None,))\n",
    "    embedding_layer.set_weights([embedding_matrix])\n",
    "    return embedding_layer\n",
    "\n",
    "embeddings_encoder = make_embedding_layer(encoder_vocab_size, wordtoix_enc, embedding_dim=embedding_dim, glove=True)\n",
    "embeddings_decoder = make_embedding_layer(decoder_vocab_size, wordtoix_dec, embedding_dim=embedding_dim, glove=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### transformer layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyper-params\n",
    "init_lr = 1e-3\n",
    "lmbda_l2 = 0.1\n",
    "d_out_rate = 0.1 # tested 0.4, 0.3, 0.1 values this 0.1 seems to be the best\n",
    "num_layers = 4 # chaged from 4 to 5 to learn better\n",
    "d_model = embedding_dim # d_model is the representation dimension or embedding dimension of a word (usually in the range 128–512)\n",
    "dff = 512 # number of neurons in feed forward network\n",
    "num_heads = 5 # first it was 8 i chenged it to 10 to use embd =300d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The Point-wise feed-forward network block is essentially a \n",
    "# two-layer linear transformation which is used identically throughout the model\n",
    "def point_wise_feed_forward_network(d_model, dff):\n",
    "    return krs.Sequential([\n",
    "        krs.layers.Dense(dff, activation='relu',kernel_regularizer=krs.regularizers.l2(l=lmbda_l2)),\n",
    "        krs.layers.Dense(d_model,kernel_regularizer=krs.regularizers.l2(l=lmbda_l2))\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class EncoderLayer(krs.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, dff, rate=d_out_rate):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "\n",
    "        self.mha = MultiHeadAttention(d_model, num_heads)\n",
    "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "\n",
    "        self.layernorm1 = krs.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = krs.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.dropout1 = krs.layers.Dropout(rate)\n",
    "        self.dropout2 = krs.layers.Dropout(rate)\n",
    "   \n",
    "    # it has 1 layer of multi-headed attention\n",
    "    def call(self, x, training, mask):\n",
    "        attn_output, _ = self.mha(x, x, x, mask)\n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(x + attn_output)\n",
    "\n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output, training=training)\n",
    "        out2 = self.layernorm2(out1 + ffn_output)\n",
    "\n",
    "        return out2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderLayer(krs.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, dff, rate=d_out_rate):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "\n",
    "        self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
    "        self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
    "\n",
    "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "\n",
    "        self.layernorm1 = krs.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = krs.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm3 = krs.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.dropout1 = krs.layers.Dropout(rate)\n",
    "        self.dropout2 = krs.layers.Dropout(rate)\n",
    "        self.dropout3 = krs.layers.Dropout(rate)\n",
    "    \n",
    "    # it has 2 layers of multi-headed attention\n",
    "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
    "        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)\n",
    "        attn1 = self.dropout1(attn1, training=training)\n",
    "        out1 = self.layernorm1(attn1 + x)\n",
    "\n",
    "        attn2, attn_weights_block2 = self.mha2(enc_output, enc_output, out1, padding_mask)\n",
    "        attn2 = self.dropout2(attn2, training=training)\n",
    "        out2 = self.layernorm2(attn2 + out1)\n",
    "\n",
    "        ffn_output = self.ffn(out2)\n",
    "        ffn_output = self.dropout3(ffn_output, training=training)\n",
    "        out3 = self.layernorm3(ffn_output + out2)\n",
    "\n",
    "        return out3, attn_weights_block1, attn_weights_block2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(krs.layers.Layer):\n",
    "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, maximum_position_encoding, rate=d_out_rate):\n",
    "        super(Encoder, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        self.embedding = embeddings_encoder\n",
    "        self.pos_encoding = positional_encoding(maximum_position_encoding, self.d_model)\n",
    "\n",
    "        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n",
    "\n",
    "        self.dropout = krs.layers.Dropout(rate)\n",
    "        self.dropout_embd = krs.layers.Dropout(rate)\n",
    "        \n",
    "    def call(self, x, training, mask):\n",
    "        seq_len = tf.shape(x)[1]\n",
    "\n",
    "        x = self.embedding(x)\n",
    "        x = self.dropout_embd(x, training=training) # dropout added to encoder input changed from nothing to this\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "        x = self.dropout(x, training=training)\n",
    "    \n",
    "        for i in range(self.num_layers):\n",
    "            x = self.enc_layers[i](x, training, mask)\n",
    "    \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(krs.layers.Layer):\n",
    "    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size, maximum_position_encoding, rate=d_out_rate):\n",
    "        super(Decoder, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        self.embedding = embeddings_decoder\n",
    "        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
    "\n",
    "        self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)] # a list of decoder layers\n",
    "        self.dropout = krs.layers.Dropout(rate)\n",
    "    \n",
    "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
    "        seq_len = tf.shape(x)[1]\n",
    "        attention_weights = {}\n",
    "\n",
    "        x = self.embedding(x)\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "        x = self.dropout(x, training=training)\n",
    "\n",
    "        for i in range(self.num_layers):\n",
    "            x, block1, block2 = self.dec_layers[i](x, enc_output, training, look_ahead_mask, padding_mask) # enc_output is fed into it\n",
    "\n",
    "            attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n",
    "            attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n",
    "    \n",
    "        return x, attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transformer(krs.Model):\n",
    "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n",
    "                                     target_vocab_size, pe_input, pe_target, rate=d_out_rate):\n",
    "        super(Transformer, self).__init__()\n",
    "\n",
    "        self.encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, pe_input, rate)\n",
    "\n",
    "        self.decoder = Decoder(num_layers, d_model, num_heads, dff, target_vocab_size, pe_target, rate)\n",
    "\n",
    "        self.final_layer = krs.layers.Dense(target_vocab_size, kernel_regularizer=krs.regularizers.l2(l=lmbda_l2))\n",
    "        \n",
    "        \n",
    "    # training argument is used in dropout inputs\n",
    "    def call(self, inp, tar, training, enc_padding_mask, look_ahead_mask, dec_padding_mask):\n",
    "        enc_output = self.encoder(inp, training, enc_padding_mask)\n",
    "       \n",
    "        dec_output, attention_weights = self.decoder(tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
    "\n",
    "        final_output = self.final_layer(dec_output)\n",
    "\n",
    "        return final_output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformer = Transformer(\n",
    "    num_layers, \n",
    "    d_model, \n",
    "    num_heads, \n",
    "    dff,\n",
    "    encoder_vocab_size, \n",
    "    decoder_vocab_size, \n",
    "    pe_input=max_len_news,\n",
    "    pe_target=max_len_summary,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Masking\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Padding mask for masking \"pad\" sequences so \n",
    "# they won't affect the loss\n",
    "def create_padding_mask(seq):\n",
    "    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
    "    return seq[:, tf.newaxis, tf.newaxis, :]\n",
    "\n",
    "# Lookahead mask for masking future words from\n",
    "# contributing in prediction of current words in self attention\n",
    "def create_look_ahead_mask(size):\n",
    "    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
    "    return mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this function is use in training step\n",
    "def create_masks(inp, tar):\n",
    "    enc_padding_mask = create_padding_mask(inp)\n",
    "    dec_padding_mask = create_padding_mask(inp)\n",
    "\n",
    "    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
    "    dec_target_padding_mask = create_padding_mask(tar)\n",
    "        \n",
    "    combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
    "  \n",
    "    return enc_padding_mask, combined_mask, dec_padding_mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_schedule = krs.optimizers.schedules.ExponentialDecay(\n",
    "    initial_learning_rate=init_lr, # originally was 1e-5\n",
    "    decay_steps=4000, # approximately 5 epochs\n",
    "    decay_rate=0.95) # originally was 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "optimizer2 = Adam(lr_schedule , beta_1=0.9, beta_2=0.98, epsilon=1e-9) # changed to init\n",
    "loss_object = SparseCategoricalCrossentropy(from_logits=True, reduction='none') # added softmax changed from_logits to false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_function(real, pred, l2= False):\n",
    " \n",
    "    if l2:\n",
    "        lambda_ = 0.0001\n",
    "        l2_norms = [tf.nn.l2_loss(v) for v in transformer.trainable_variables]\n",
    "        l2_norm = tf.reduce_sum(l2_norms)\n",
    "        l2_value = lambda_ * l2_norm\n",
    "        loss_ = loss_object(real, pred) + l2_value\n",
    "    else:\n",
    "        loss_ = loss_object(real, pred) \n",
    "    \n",
    "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "    \n",
    "    return tf.reduce_sum(loss_)/tf.reduce_sum(mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_path4 =\"checkpoints4\"\n",
    "\n",
    "ckpt4 = tf.train.Checkpoint(transformer=transformer, optimizer=optimizer2)\n",
    "\n",
    "ckpt_manager4 = tf.train.CheckpointManager(ckpt4, checkpoint_path4, max_to_keep=100)\n",
    "\n",
    "# if ckpt_manager4.latest_checkpoint:\n",
    "#     ckpt4.restore(ckpt_manager4.latest_checkpoint)\n",
    "#     print ('Latest checkpoint restored!!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(input_document):\n",
    "    input_document = document_tokenizer.texts_to_sequences([input_document])\n",
    "    input_document = krs.preprocessing.sequence.pad_sequences(input_document, maxlen=max_len_news, \n",
    "                                                                           padding='post', truncating='post')\n",
    "    \n",
    "    encoder_input = tf.expand_dims(input_document[0], 0)\n",
    "\n",
    "    decoder_input = [summary_tokenizer.word_index[start_token]]\n",
    "    output = tf.expand_dims(decoder_input, 0)\n",
    "    \n",
    "    for i in range(max_len_summary):\n",
    "        enc_padding_mask, combined_mask, dec_padding_mask = create_masks(encoder_input, output)\n",
    "\n",
    "        predictions, attention_weights = transformer(\n",
    "            encoder_input, \n",
    "            output,\n",
    "            False,\n",
    "            enc_padding_mask,\n",
    "            combined_mask,\n",
    "            dec_padding_mask\n",
    "        )\n",
    "\n",
    "        predictions = predictions[: ,-1:, :]\n",
    "        predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
    "        # stop prediciting if it reached end_token\n",
    "        if predicted_id == summary_tokenizer.word_index[end_token]:\n",
    "            return tf.squeeze(output, axis=0), attention_weights\n",
    "\n",
    "        output = tf.concat([output, predicted_id], axis=-1)\n",
    "    return tf.squeeze(output, axis=0), attention_weights\n",
    "\n",
    "def summarize(input_document):\n",
    "    # not considering attention weights for now, can be used to plot attention heatmaps in the future\n",
    "    summarized = evaluate(input_document=input_document)[0].numpy()\n",
    "    summarized = np.expand_dims(summarized[1:], 0)  # remove start_token\n",
    "    return summary_tokenizer.sequences_to_texts(summarized)[0]  # since there is just one translated document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def validate():\n",
    "    print('validation started ...')\n",
    "    val_loss.reset_states()\n",
    "    for (batch, (inp, tar)) in enumerate(dataset_val):    \n",
    "        tar_inp = tar[:, :-1] # <startseq> hi im moein\n",
    "        tar_real = tar[:, 1:] # hi im moein <endseq>\n",
    "\n",
    "        enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
    "\n",
    "        # Operations are recorded if they are executed within this context manager\n",
    "        # and at least one of their inputs is being \"watched\". Trainable variables are automatically watched\n",
    "        predictions, _ = transformer(\n",
    "            inp, tar_inp, \n",
    "            False, \n",
    "            enc_padding_mask, \n",
    "            combined_mask, \n",
    "            dec_padding_mask\n",
    "        )\n",
    "        loss = loss_function(tar_real, predictions)\n",
    "        val_loss(loss)\n",
    "    print('\\n* Validation loss: {} '.format(val_loss.result()) )\n",
    "    return val_loss.result()\n",
    "# validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function # Compiles a function into a callable TensorFlow graph\n",
    "def train_step(inp, tar):\n",
    "    tar_inp = tar[:, :-1] # <startseq> hi im moein\n",
    "    tar_real = tar[:, 1:] # hi im moein <endseq>\n",
    "\n",
    "    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
    "    \n",
    "    # Operations are recorded if they are executed within this context manager\n",
    "    # and at least one of their inputs is being \"watched\". Trainable variables are automatically watched\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions, _ = transformer(\n",
    "            inp, tar_inp, \n",
    "            True, \n",
    "            enc_padding_mask, \n",
    "            combined_mask, \n",
    "            dec_padding_mask\n",
    "        )\n",
    "        loss = loss_function(tar_real, predictions)\n",
    "\n",
    "    gradients = tape.gradient(loss, transformer.trainable_variables)    \n",
    "    optimizer2.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
    "    \n",
    "    # mean the loss with new computed  loss of the step\n",
    "    train_loss(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "history={'val':[],'train':[]}\n",
    "EPOCHS = 300\n",
    "not_progressing = 0\n",
    "# Computes the (weighted) mean of the given loss values.\n",
    "train_loss = krs.metrics.Mean(name='train_loss')\n",
    "val_loss = krs.metrics.Mean(name='val_loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lmbda_l2': 0.1,\n",
       " 'd_out_rate': 0.15,\n",
       " 'num_layers': 5,\n",
       " 'd_model': 50,\n",
       " 'dff': 512,\n",
       " 'num_heads': 5,\n",
       " 'init_lr': 0.001}"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {\n",
    "'lmbda_l2' : lmbda_l2,\n",
    "'d_out_rate' :d_out_rate,\n",
    "'num_layers' : num_layers ,\n",
    "'d_model' : d_model  ,\n",
    "'dff' : dff ,\n",
    "'num_heads' : num_heads,\n",
    "'init_lr':init_lr}\n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [],
   "source": [
    "ep = 1\n",
    "best_val_loss = np.inf\n",
    "i1,i2,i3,i4 = np.random.randint(len(summaries_val)),np.random.randint(len(summaries_val)),np.random.randint(len(summaries_val)),np.random.randint(len(summaries_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(params)\n",
    "print('#'*40)\n",
    "\n",
    "for epoch in range(ep,EPOCHS+1):\n",
    "    ep = epoch\n",
    "    start = time.time()\n",
    "\n",
    "    train_loss.reset_states()\n",
    "  \n",
    "    for (batch, (inp, tar)) in enumerate(dataset):\n",
    "        \n",
    "        train_step(inp, tar)\n",
    "    \n",
    "        if batch % 150 == 0:\n",
    "            print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch , batch, train_loss.result()))\n",
    "                  \n",
    "    print()\n",
    "    print(summarize(clean_words(longreview_val['long'][i1])))\n",
    "    print(summarize(clean_words(longreview_val['long'][i2])))\n",
    "    print(summarize(clean_words(longreview_val['long'][i3])))\n",
    "    print(summarize(clean_words(longreview_val['long'][i4])))\n",
    "    print()\n",
    "    \n",
    "    val_loss_ = validate().numpy()\n",
    "    history['val'].append((epoch,val_loss_))\n",
    "    print ('\\n* Train Loss {:.4f}'.format(train_loss.result()))\n",
    "    history['train'].append((epoch,train_loss.result().numpy()))\n",
    "    \n",
    "    \n",
    "    if best_val_loss-val_loss_ > 0.1:\n",
    "        ckpt_save_path4 = ckpt_manager4.save()\n",
    "        print ('\\nSaving checkpoint for epoch {} at {}'.format(epoch, ckpt_save_path4))  \n",
    "        best_val_loss = val_loss_\n",
    "    \n",
    "    hist(history)\n",
    "    print('Current Lr: ',optimizer2._decayed_lr('float32').numpy())\n",
    "    print ('\\nTime taken for this epoch: {:.2f} secs\\n'.format(time.time() - start))\n",
    "    print('='*40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "output removed to reduce file size for github"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3xU9Z3/8ddnLrkNSYAkYiAoEBQUDAQjUryA4kPFqrXWIqu2q21lobXadttV299a6/562/pjLWUrq713pcri3bXWegG1UCQIRESKBKEJQUgC5kZCLvP9/XHOGSaTmVzIJJMz+Twfjzwyc86Zme/JwPt8z/f7Pd8jxhiUUkq5nyfRBVBKKRUfGuhKKZUkNNCVUipJaKArpVSS0EBXSqkkoYGulFJJQgNdKaWShAa6Snoisk9ELkt0OZQaaBroSimVJDTQ1bAlIreLyB4ROSIiz4nIWHu5iMh/iMhhEakTkTIRmW6vu0pEdopIg4gcEJFvJnYvlDpBA10NSyJyKfBDYBGQD+wHHrdXXw5cDJwJjARuBGrtdb8E/skYkwlMB14bxGIr1S1fogugVILcDPzKGPMOgIjcCxwVkQlAG5AJTAXeNsa8H/a6NuBsEdlujDkKHB3UUivVDa2hq+FqLFatHABjTCNWLXycMeY1YCXwn8AhEXlERLLsTT8DXAXsF5H1IvKJQS63UjFpoKvhqgo43XkiIgEgBzgAYIxZYYw5F5iG1fTyLXv5ZmPMp4BTgGeANYNcbqVi0kBXw4VfRNKcH6wgvk1EZopIKvADYJMxZp+InCci54uIH2gCWoAOEUkRkZtFJNsY0wbUAx0J2yOlImigq+HiRaA57Oci4F+BJ4GDQCGw2N42C3gUq318P1ZTzIP2us8B+0SkHlgK3DJI5VeqR6I3uFBKqeSgNXSllEoSGuhKKZUkNNCVUipJaKArpVSSSNiVorm5uWbChAmJ+nillHKlLVu21Bhj8qKtS1igT5gwgdLS0kR9vFJKuZKI7I+1TptclFIqSWigK6VUktBAV0qpJKHT5yql4qKtrY3KykpaWloSXZSkkJaWRkFBAX6/v9ev0UBXSsVFZWUlmZmZTJgwARFJdHFczRhDbW0tlZWVTJw4sdevc1eTS9ka+I/pcP9I63eZzlyq1FDR0tJCTk6OhnkciAg5OTl9PttxTw29bA08fye0NVvP6yqs5wBFixJXLqVUiIZ5/JzM39I9NfRXHzgR5o62Zmu5UkopFwV6XWXfliulVDdGjBgBQFVVFTfccEPUbebPn9/jBZAPPfQQx44dCz2/6qqr+Pjjj+NX0D5wT6BnF/RtuVJqyFq1vpwN5TWdlm0or2HV+vJBL8vYsWNZu3btSb8+MtBffPFFRo4cGY+i9Zl7An3BfeBP77zMn24tV0q5SlFBNnes3hoK9Q3lNdyxeitFBdkn/Z533303P//5z0PP77//fr73ve+xYMECZs2axTnnnMOzzz7b5XX79u1j+vTpADQ3N7N48WKKioq48cYbaW4+0cy7bNkySkpKmDZtGt/97ncBWLFiBVVVVVxyySVccsklgDWtSU2NtV/Lly9n+vTpTJ8+nYceeij0eWeddRa3334706ZN4/LLL+/0Of3hnk5Rp+PzT9+GpmoI5MEVP9AOUaWGoO89/x47q+q73eaUzFQ+/8u3GZOVyqH640w+ZQQ/feUDfvrKB1G3P3tsFt+9ZlrM91u8eDFf+9rX+PKXvwzAmjVreOmll/j6179OVlYWNTU1zJkzh2uvvTZmh+PDDz9MRkYGZWVllJWVMWvWrNC673//+4wePZqOjg4WLFhAWVkZd955J8uXL+f1118nNze303tt2bKFX//612zatAljDOeffz7z5s1j1KhRfPDBB/zhD3/g0UcfZdGiRTz55JPcckv/72bonho6WOG9eLX1+LpVGuZKuVh2up8xWakc+LiFMVmpZKf3/gKaaIqLizl8+DBVVVVs376dUaNGkZ+fz7e//W2Kioq47LLLOHDgAIcOHYr5Hm+88UYoWIuKiigqKgqtW7NmDbNmzaK4uJj33nuPnTt3dluet956i09/+tMEAgFGjBjB9ddfz5tvvgnAxIkTmTlzJgDnnnsu+/bt69e+O3pVQxeRkcAvgOmAAb5gjNkYtn4+8Czwob3oKWPMwAw/8dpfekfrgLy9Uqr/uqtJO5xmljsvncx/b/o7d112BnMLc3t8XXduuOEG1q5dy0cffcTixYt57LHHqK6uZsuWLfj9fiZMmNDj2O5otfcPP/yQBx98kM2bNzNq1ChuvfXWHt+nu/s1p6amhh57vd64Nbn0tob+U+AlY8xUYAbwfpRt3jTGzLR/Bm4sodf+Q3QcH7CPUEoNLCfMV95UzDcun8LKm4o7tamfrMWLF/P444+zdu1abrjhBurq6jjllFPw+/28/vrr7N8fc+ZZAC6++GIee+wxAHbs2EFZWRkA9fX1BAIBsrOzOXToEH/84x9Dr8nMzKShoSHqez3zzDMcO3aMpqYmnn76aS666KJ+7V9Peqyhi0gWcDFwK4AxphVIXPXYZwd6u9bQlXKrsso6Vt5UHKqRzy3MZeVNxZRV1vWrlj5t2jQaGhoYN24c+fn53HzzzVxzzTWUlJQwc+ZMpk6d2u3rly1bxm233UZRUREzZ85k9uzZAMyYMYPi4mKmTZvGpEmTuOCCC0KvWbJkCQsXLiQ/P5/XX389tHzWrFnceuutoff40pe+RHFxcdyaV6KR7k4LAERkJvAIsBOrdr4FuMsY0xS2zXzgSaASqAK+aYx5L8p7LQGWAJx22mnn9nS0jOrjv8ND58C1K2HW5/r+eqXUgHj//fc566yzEl2MpBLtbyoiW4wxJdG2702Tiw+YBTxsjCkGmoB7IrZ5BzjdGDMD+BnwTLQ3MsY8YowpMcaU5OVFvYNSz7TJRSmloupNoFcClcaYTfbztVgBH2KMqTfGNNqPXwT8ItK/3o1YfCnWb21yUUqpTnoMdGPMR0CFiEyxFy3Aan4JEZFTxe4aFpHZ9vvWxrmsFq2hK6VUVL29sOirwGMikgLsBW4TkaUAxphVwA3AMhFpB5qBxaanxvmT5bVr6B1tA/L2SinlVr0KdGPMNiCyEX5V2PqVwMo4lis2rw/EA+1aQ1dKqXDuulLU4U3VJhellIrg0kBP0SYXpVQnH3/8cafJuXorkdPdxps7A92Xok0uSrldnG8pGSvQOzo6un1dIqe7jTf3zLYYzpuqc7ko5WYDcEvJe+65h/LycmbOnInf72fEiBHk5+ezbds2du7cyXXXXUdFRQUtLS3cddddLFmyBLCmuy0tLaWxsZGFCxdy4YUXsmHDBsaNG8ezzz5Lenp6D588dLgz0LWGrtTQ9sd74KN3Y6+v3Ny1H6ytGZ69A7b8NvprTj0HFv4o5lv+6Ec/YseOHWzbto1169bxyU9+kh07djBx4kQAfvWrXzF69Giam5s577zz+MxnPkNOTk6n9xioaW0HizsD3ZuiNXSl3CzWoIY4DnaYPXt2KMzBuhnF008/DUBFRQUffPBBl0AfqGltB4sGulIq/rqpSQNWm3ldRdfl2ePhtv+NSxECgUDo8bp163jllVfYuHEjGRkZzJ8/P+r0twM1re1gcWmnaKo2uSjlZgNwS8lY09gC1NXVMWrUKDIyMti1axd//etfT/pzhjIX19B12KJSruV0fL76ANRVWjd7X3Bfv+5ClpOTwwUXXMD06dNJT09nzJgxoXVXXnklq1atoqioiClTpjBnzpz+7sGQ1OP0uQOlpKTElJaWntyLf3cdtDbCl16Jb6GUUidNp8+Nv4GYPnfo0SYXpZTqwp2Brp2iSinVhQa6UipuEtWEm4xO5m/pzkD3peoNLpQaYtLS0qitrdVQjwNjDLW1taSlpfXpdS4e5aJt6EoNJQUFBVRWVlJdXZ3ooiSFtLQ0CgoK+vQaFwe61tCVGkr8fn+nKzPV4HNpk0uKNrkopVQE1wT6qvXlbCivsZ7YN7jYUF7DqvXliS2YUkoNEa4J9KKCbO5YvdUKdW8KmCB3PVZKUUF2ooumlFJDgmsCfW5hLitvKmbZf7/D4+8cAuBnN05jbmFugkumlFJDQ68CXURGishaEdklIu+LyCci1ouIrBCRPSJSJiKzBqKwcwtzmVuYw+5aq/18zmkjBuJjlFLKlXpbQ/8p8JIxZiowA3g/Yv1C4Az7ZwnwcNxKGGZDeQ1vfVBDqz04Z3P5RwPxMUop5Uo9BrqIZAEXA78EMMa0GmMi76j6KeB3xvJXYKSI5MezoBvKa7hj9VZ+ds4e/tn3PwCctvYqdv/5l/H8GKWUcq3e1NAnAdXAr0Vkq4j8QkQCEduMA8Jnq6+0l8VNWWUdj8/5Oxfv+jdGSSMAYzjCxI3f7vfNZZVSKhn0JtB9wCzgYWNMMdAE3BOxjUR5XZfrf0VkiYiUikhpX68mWzqvkDN3/Aee9s53EPEHW6w5lZVSapjrTaBXApXGmE3287VYAR+5zfiw5wVAVeQbGWMeMcaUGGNK8vLy+l7ausq+LVdKqWGkx0A3xnwEVIjIFHvRAmBnxGbPAZ+3R7vMAeqMMQfjW1Ssu5r0ZblSSg0jvR3l8lXgMREpA2YCPxCRpSKy1F7/IrAX2AM8Cnw57iUFWHAfJs73IVRKqWTRq8m5jDHbgMhbHq0KW2+Ar8SxXNEVLUKAQ0/+C2PkKKSPhoU/7td9CJVSKlm45krRkKJF3OJ70Hp8ybc1zJVSyua+QAdMin2F6PGGxBZEKaWGEFcGui8lnQ48GuhKKRXGlYEeSPPTLAFobUx0UZRSashwZaBnpHg5JmlwXANdKaUcrgz0QIqPJjLgeH2ii6KUUkOGKwM9I9VLI2na5KKUUmFcGeiBFB8NwXTtFFVKqTCuDPSMFC/1wVRtQ1dKqTAuDXQf9cE0jDa5KKVUiKsCfdX6cjaU1xBI9dJIOhyvZ0N5DavWlye6aEoplXCuCvSigmzuWL2Vg3UtVqfo8UbueOwdigqyE100pZRKOFcF+tzCXFbeVMzjm/9Oo0lHMPx80RTmFuYmumhKKZVwrgp0sEL9W/llLPM9B8Cc/71Sb0GnlFL0cvrcoWT3n3/JjR/9hHRptRY0VMHzd1qPdeZFpdQw5qoa+obyGkb85Qek09p5RVuz3ldUKTXsuSrQyyrryKc2+kq9r6hSaphzVaAvnVeI6H1FlVIqKlcFOqD3FVVKqRjcF+hFi5BrVlCFPVQxNRuuWaEdokqpYc99gQ5QtIhF6Y/S4kmH4ls0zJVSil4GuojsE5F3RWSbiJRGWT9fROrs9dtEZMDbP7LS/DRJJjQfGeiPUkopV+jLOPRLjDE13ax/0xhzdX8L1FtZ6T7qGjLJaT46WB+plFJDmjubXIDMND91jAANdKWUAnof6AZ4WUS2iMiSGNt8QkS2i8gfRWRatA1EZImIlIpIaXV19UkV2JGV5udIMKCBrpRStt42uVxgjKkSkVOAP4vILmPMG2Hr3wFON8Y0ishVwDPAGZFvYox5BHgEoKSkxPSn4FnpPmo7MjTQlVLK1qsaujGmyv59GHgamB2xvt4Y02g/fhHwi8iAToGYleanuiMD03wUTL+ODUoplRR6DHQRCYhIpvMYuBzYEbHNqSIi9uPZ9vvGuEa/f5ybXMz8+M/c7H0FCbZz/Cdn8coTPxuIj1NKKdfoTZPLGOBpO699wGpjzEsishTAGLMKuAFYJiLtQDOw2JiBqTYXFWTzwn//lPvlEVKkBYDUYwe5ZPf3oWyMjklXSg1bMkC526OSkhJTWtplSHuvtPzkLNKaqrquyB4PX9/RdblSSiUJEdlijCmJts6VwxbTmg5GX6EzLiqlhjFXBnpLID/6Cp1xUSk1jLku0DeU1/DAsRvo8KZ1Wt7h1RkXlVLDm+sCvayyjqtvuQvvp37GAZOLAdp9AV4/8zvaIaqUGtZcd0/RpfMK7UeLuOlPeTzZuozcqRdx2We+mtByKaVUormuhh4ud0QqRxkJjYcSXRSllEo4Vwd63ohUDpssaOrfvDBKKZUM3B3omakcbM+ExsOJLopSSiWcKwPdufz/gubXuNz8BXOshpZ/18v/lVLDmysD3bn8/9IPvk+WNCNA2rEq+/L/NYkunlJKJYQrA31uYS73ZawlJdjSabm3oxlefSBBpVJKqcRyZaCDXv6vlFKRXBvoevm/Ukp15spAP3H5f3qn5Xr5v1JqOHNloJ+4/H8FhyQPA3T4MvTyf6XUsOa6S/+h8+X/974zkbsrljHl9PFcdqNe/q+UGr5cWUOHE2PRC0alU9ExCuqr2FBew6r15YkumlJKJYRrA72oIJs7Vm+lvSOIr6MZU/035vx+Mre+fbWORVdKDUuuDfS5hbmsvKmY9u1rmOvZiWDwYKxb0z1/p4a6UmrYcW2ggxXq96asIUU6Oq9o0wuMlFLDj6sDfUN5DdltMSbm0guMlFLDTK8CXUT2ici7IrJNREqjrBcRWSEie0SkTERmxb+onW0or+GO1Vtp1QuMlFIK6FsN/RJjzExjTEmUdQuBM+yfJcDD8Shcd8oq61h5UzFpV3yPFlI7r/TrBUZKqeEnXk0unwJ+Zyx/BUaKSIyqc3wsnVfI3MJcKFrEr0Z/jXa81ors8XDNCr3ASCk17PQ20A3wsohsEZElUdaPAyrCnlfayzoRkSUiUioipdXV8bnL0Kr15ZRmXcYmKYJTi+DrO9gQuFTHoyulhp3eBvoFxphZWE0rXxGRiyPWS5TXmC4LjHnEGFNijCnJy8vrY1GjKyrIZmP5ETra2zAfvYu5fyQTfn8+l7aui8v7K6WUW/Qq0I0xVfbvw8DTwOyITSqB8WHPC4CqeBSwJ3MLc/n3M3cxx/M+gkEwjKWGM9/+jo5FV0oNKz0GuogERCTTeQxcDuyI2Ow54PP2aJc5QJ0xJsaE5fF3+UeP6Fh0pdSw15vJucYAT4uIs/1qY8xLIrIUwBizCngRuArYAxwDbhuY4kaX0hTjZEDHoiulhpEeA90YsxeYEWX5qrDHBvhKfIvWOxvKa5hADmOp6bpSx6IrpYYRV18pCtZ49DfGL+syFr1VUnll7D8lqFRKKTX4XB/oS+cVctr8W/lO8HaajR8DtATGcj//REbJPyS6eEopNWhcH+hgjXQ5de7n2Bg8G4CUpoPcl7GWuU2vJbhkSik1eFx5x6Jobgm8TY7nPQSQ8Gl0Qa8aVUoNC0lRQwcYtfGHpEh754U6dFEpNYwkRaBvKK8hpSn6sHejQxeVUsNEUgR6WWUdTemnRl13PNb0ukoplWSSItCXzisk86oHaPOkdVpuENLOWpigUiml1OBKikAHoGgRwRn/gAmbEkwwsH21zumilBoWkifQAbP7T0jkvI/aMaqUGiaSJtA3lNeQ0qgdo0qp4StpAl07RpVSw13SBLp2jCqlhrukCXQAihaxdfRVXTpGO7au5pUnfpa4ciml1CBIrkAHzmrY2KVj1NvRzIV//3liCqSUUoMkqQJ9Q3kNgeMfRV2XFuNKUqWUShZJFehllXW0xuoATR81uIVRSqlBllSBvnReIWlXfI92vF3WBY83aDu6UiqpJVWgA2wIXEoDGV2We4Jt2o6ulEpqSRfoZZV1jKQx6jptR1dKJbOkC/Sl8wqRGDeHbk3JZtX68kEukVJKDY5eB7qIeEVkq4i8EGXdrSJSLSLb7J8vxbeYfbTgPtqj3YyptZFLW9cNenGUUmow9KWGfhfwfjfrnzDGzLR/ftHPcvWL1Y6e3mV5Cu2ctu1BraUrpZJSrwJdRAqATwIJDere6q4dPaWpiqKC7EEukVJKDbze1tAfAv4FCHazzWdEpExE1orI+GgbiMgSESkVkdLq6uq+lrXXumtHF4S5Ta8N2GcrpVSi9BjoInI1cNgYs6WbzZ4HJhhjioBXgN9G28gY84gxpsQYU5KXl3dSBe6t3dO/HvXoIxha/vRdbXZRSiWd3tTQLwCuFZF9wOPApSLy3+EbGGNqjTHH7aePAufGtZQn4bWU+UTe68KR0lSF14OGulIqqfQY6MaYe40xBcaYCcBi4DVjzC3h24hI+PX219J95+mgWDqvkOOBsVHXCfDha7/RtnSlVFI56XHoIvKAiFxrP71TRN4Tke3AncCt8Shcf7112pcxUerpAtzv/x1zC3MHv1BKKTVAxIRPHj6ISkpKTGlp6cB/0P2xa+G7L1jOaynzWTqvcODLoZRScSAiW4wxJdHWJd2VopFaYjS7AOT95T5tS1dKJY2kD3Sr2SW6kTTy4Wu/0VBXSiWFpA/0y278Ki2+6M0uAvyr97c8vG6vhrpSyvWSPtAB0q99MOa6tPY6HpyyS0NdKeV6wyLQKVpEW2r0OxYJULzzRyybP4mH1+3VoYxKKdcaHoEOrJ/0zz22pa+8qViHMiqlXGvYBHpPben38GvmFuayobxGm12UUq40bAIdrLb0WLX0zGADG1f8I3es3qpt6UopVxpWgU7RIiR9dNRVInB+7TMsHVUa6iC996ky7n2qTMNdKeUKwyvQARb+OOYqj8BnDq+keHw2K17dwzNbD/BC2UGtsSulXGH4BXrRIohRSwcYLY1cvOfHNLS04/N6uHPBZB3SqJRyheEX6GDX0qNPrisCn/e+wvd8vyLFKzy8bm9oSKOGulJqKBuegV60CEq+ELOD1An1O4//F8YYDXWllCsMz0AHuHp5zA5SOBHqX2v9Lxqa21jxyp5OFx/p8Eal1FAzfAMdum16ASvUP+d9hYW8ScPxdpa/vJuVNxXzXlUdX/xNqdbWlVJDSvLPh96TF74Bpb/sdpMOI3y9bRnPBS9kVIafo8famJwX4FDDce5cMJm91U2hbU/PCej86kqpAdPdfOga6NCrUDcGftdxGd9t/0JomdcjeAWCxhrymOr3cueCyXQEoaggm7LKOg13pVRcDesbXPTK1cuh5IvdbhI++sXRETS0dhjag4a2DsO1M/J5eN1eKo408cXflOpEX0qpQeVLdAGGjKuXW7+7qak7oe4R4V/bbuu0bmSGn8c2VeD3CI9tquA7n5wamhvm+e1V2hSjlBpwWkMPd/Xybi86AivUb/H8mR2pX+Baz1uh5UePtQHQFjR4BDaW1/Lom+X80++3hK421WkElFIDqdeBLiJeEdkqIi9EWZcqIk+IyB4R2SQiE+JZyEG18Mfg8Xe7iQiMkBZ+6v85D4Q1wTiCBl7bVc33/3cXx463c96EUSx/eXenaQR02KNSKt76UkO/C3g/xrovAkeNMZOB/wBiT5gy1BUtgut+Dv5Aj5uKwOd8r/BQ4Peh55E67HBvbgsyJjOVn7y0m80f1oaGPeoEYEqpeOnVKBcRKQB+C3wf+IYx5uqI9X8C7jfGbBQRH/ARkGe6efMhNcolll6MfgFrBEybL4N9c/4vV7x6atQrUP0eoS14Ys3Y7DQONxzHI4AInz13HDuq6mlsaWf2xBPNPtfMGBtqg9eRM0qpfg9bFJG1wA+BTOCbUQJ9B3ClMabSfl4OnG+MqYn1nq4IdOh1qAMYoMmk8e22L1A26nIO1bfQ3Bbs8XXF47PZWlGHxx4C6fcKxljvZ4zB5/Xw2XPH8ccdh1g2f5IOi1RqGOvXsEURuRo4bIzZ0t1mUZZ1OVKIyBIRKRWR0urq6p4+emjoxZBGh2C3raf8nNvqViIi3Hz+ePzeE3+eQIq3y+u2VtQBVpiPyUyhzR4K2RE0BA2k+ayRM4Jhxat78Hrgi78ppeJIE/c+Vcatv36bDeU1ndrltY1eqeGnxxq6iPwQ+BzQDqQBWcBTxphbwrZJziaXcGVr4PmvQVtTz9tyorb+b3I7vhmLWL2pgpH2VaZOTVyIctSLELmNCHhFmHdmLq/uqsYrMH9KHq//zTpAFhVkI8Cujxr5xuVnhK5iPT0nEFofft/UDeU1WtNXykXidqWoiMwnepPLV4BzjDFLRWQxcL0xZlF37+W6QHf0oQkGTgT7i6d9i3+rOIeW1g7agibUzBIpze+hpS3Yq7B3DgzRTM4LsK/2GMYYUnweriseh0dg7ZYDFIxKZ/bE0Ryqb2H932q456opnaYvOFTfwuyJOaFmHW3eUWroGJBAF5EHgFJjzHMikgb8HigGjgCLjTF7u3sv1wY69DnU4UQ4H/Nm879j7+T/7J2GCASNIW9EKlV1LaFtnaD2ijVKBmDqmBHsOtTIyAw/H9tj3nvL5xHOGZfF1oq60IHC+Z3q89AeNBhjEBGCQYPB6rQ9VH+cibkZ/P1IM9+68kz2Vjex82A9OYEUZk/MYem8QlatL8frgY4gocDXWr9SA0fnchkIfWyCCWeA4550nhn3TRrO/DTLX/6AuYWjWbe7hpHpPhpaOjhtdDp7qpvweSTUMXrh5Bxe3VWNzyO0h1XNw4M/d0QKNY2tJ9bZYRvL2WMz2VnV0GlZ3gg/1Y1todD32F0A9oAc2oOwYGoep2SlsetgPVsr6picF+C8iaO5ZsZYvvibUqacOoKGlnZGpPm4+8qpPL+9ikP1LYzJSgPQUTtKnSQN9IHUj2AHaPVawx1fS5kfat9+9M1ylr/8AZ8oHB0KwGe2HgBARJgzaTRv7q7pNAwyXHdNMd2t660ZBVlsr6wPPXeaj5yu38K8AHvsJhy/R+gwJjRqxyPgEWFCjlXzv/CMHIDQfh6sa+GCyTn8ZU8tSy6eRFllHW9/WEttUys5gRRqm1q5uiifF8oOhp7nBFIIGrhgck6nEUA9HTBWrS/XPgXlOhrog+EkmmE6SQnA1Q9B0aKoQXPvU2XsPFjP1UX5rHh1D8fbg1w0OYd1u6vpCILPA8KJse4Lpubx6q7oI4nGj0qnqq65U809POidx6k+D8fbex52aY3cMTS1ntjWGXrZHuXoEV7z99g1fq9YByunaWjB1Dze3FNLbiCFqrqW0JmG89spo/N7sn0QWTA1jw3lR7h+1lieeqeKX95a0unvGG5DeQ1f/E0p37j8DG6/qDD0/PpZYxk/OtCpCemRN/ay5OJJGv4q4TTQB0s/a+sh6aOtKQiKuvYrr1pfzv7aJq6ZMZayyjr21zYxKS/AC2UHAcgJpIS2fW1XdSg8x2anUVXX0qmzNTTaRkBsMaIAABGYSURBVKyLo+BE2I7JSuVQ/fFua/QjUr00Hu/o376GcT7L6S9wZKX5qG9pj/m63ICfmqY2MvwejrUFQ9vPHJ/N6EAK+2uPcXpORqdavnMBl0fgsU0V5ASsEUiTcq0Dw+S8AAZobu3gUP1x5k/JZd3uGsZkppKe4iUrzce7B+q58byC0N/nYF0LHrHONn54fRFAaHK2g3UtoQOCc8B+fnsVOw/Wh5qkIPqFZNDz6KTenG3oGUly0EAfbPEKdug23GNxappOk40TWgum5mGAdfYQR69HyBuRytFjrbR2WB2jzrLwWrHfK6Ex8Q6necUZignWdiJCa1itPtoBITPNR0M3AR0PHgGfx0NrR9eyhF/AhaFL01XkCKTQWQEQfr7inBV4xDqbmVto9XF4PcKYzFTS/B4O1rXg83q4dkY+j2+uZExmKlPzM3lrTy3YHdHtQYNgfR9tHUFSfR4+UZjDW3tquXByDvnZaTz1ThWfKBzN/tpjNLd2UNPYym++cB4AP35pFx7g3QP13L1wCrdfVBhqtvvG5WeEOqw3lNdwx+qtXDFtDNfMGAvAHau3svKm4tABxTkQQeewD69IOAclrwf2VjeFZhLt7cEh/MDivO+kvECncvZnhtJkP3BpoCdK2Rr4493QfCQ+79fLcI/8Bx0+EgUI1Q7B+g9871NlnV5/ek6gU7s1WLNHtnYEMQZOzbJq+z6P1VwSXuu/+fzxrNlc2SkkU30eOoLBTtuGB72zbNyodA7VtURtpgnfLtbzvnDKHi9OWS4JuybAsWBqHm/vO0pza7vdvGR1dAeJvg9eEUQMYh8224ImdIYV2sYjTMzJYF/tMTwCbR2GmfaByDlTcZqfnA5q5yzlvQP1dNh/40l5ATLTfOw4UI8I5I2wzkCcM5NzCrIYbf8beHN3DR3GMKMgm1GBFF6zr4M4p8A6E9pWUcfM8dnsrz0GwOyJo/nh9UVsKK/hxy/tCo2OKirI5o7VW5k53rpfwMbyWprbgtx8/ngO1rXwxu4aPB7h7PxMDHTqO3GG1zqjrYIG8rPTQvvylz21jBuZxuObKykal8UV0/Pxeuh0cAPYX2u9j3Om6xycdh6s5+z8rNDyaAeq7v5/RY70ghN9Os4BEQj18ZzMgUsDfSiIZ60dTqrmfrKcWhSc+A9QcaSJNZsrQeDssVkI1j9Sp+Y7bWwWh+uP81F9izVSBzh9dAb7a4/RYUxou/YO0yngnRE7kU0/zvPINnSHc2vACbkZ7Ks51ud9TPN7yPB7ORI2JNQJxkgj0/183Bx76GisA43fK7R19P7/2ymZKRxuaO2yPPL9U7zCZ0sKeGxTRZdtz87PZOfBE6OYIju0nfey+jxMp4NceCXA57E6s1vDyh/5HYzNTqOm8XhoG79XOH30iYNOa4dhcl6A03Iy+LCmiQ9rjuG1R3FFHsOdPqDQWaI9F5J15mUd6Fo7rOs5dhyoJ4ihI9j5rCloICfgp7apjZnjsymvbuK8CaMA2LzvKK3tQdqDhqljRvDewQZ8HvB7PaGz1aKCbBpa2qk42sxnzx1H0FgHEg9QdqCeCTkZzJ44msP1Lby6q5qZ463tnea4i8/MJT/7xMFl10cNtHUYPCJ8tmQc/1N6gFS/h//63Lkx+3mi0UAfSuId7DCo4e6IPAUHq+P27Q+PhGpmQKh2dnZ+FqfnBEIHhkP1Vm3zrT21TMvPZFQghfV2B2/x+GzKDtQjWAETPib+w5pjofbsiTkZfFh7jDGZVhNR8fhs3qtqwBgTcwSQE2BOWDvPU7yC1yO0tAU7jdOPDK3IZhtHZqqXhoj+hFjB7tQ44UQzTn/ONrr7rHjrzedEa8pK8QpzC3NYt9ua3qm7vpkUr3XgKMwNUF7TFPUzU7zCRWdYV0v3VCYBUnwSujrb5/UwNjstNBIr3KiwJkSHU1afx+q4b+swnYYKR3td8fhsdh5s4Hh7sMvZoNNnlZHi5Rf/GLvTPub+aKAPQQMR7I4EBPzJiNWW6gxZdMauR1616ow4CR+a6JwyP7P1AO0dBo9HaA8GY47kCX8e3mbu9wrTx2ZFb0OPeL1TG4wW/lPHjGD/kWNdJmcrzAtQ9XFzp+Vej9VHESvknPf32eXo7n+sc4/b3BEpVDd2rd078rPTOBjWhBOupzOQ8DLFU1+H1Pam2Sx8ltNoTYTROP1Dsbbpazljfd6dl07mG5dP6f0bOe+ngT6ExbudPRqXBHx/hTcNOQ7Vt7C/9hhZaT7KDtSHavXOBVyfLRnHE5srmT4uiwZ75IvTh+C0BY9I84XGvjfanbnOafW8M3N5Y3cNBuuqXzjxn93vEfy+E7XBE81GwuLzClhTWkl7hyHN72HcSOtCsgl2u3i0Gt97VfWdmjwmh433d6T4rJk5H9tU0SV4Ig8808dmsaOqPurIp+5Ca/rYLMqrG0MHJefagvagCR1Qop3dhJfBMTrg50iTtZ8pXsHn9XCs9eRGTkV2WufZF9mNtptdooksT7rf06sZUh3RavQ9llOspp0UX9+bW0AD3T003AeMczbg1Oqd+72e7Fw14e/nnFmseHUPVxflA7D+b9Ucqj/O4tkFoWmPXyg7SHX9cWoaW/nMueNCHWS/eHMvG+2x809srgxNvDY5L8DfjzZz0eQc3vygFoMJdVpmpfnYWlFHqs9DTiCFQw0tBIMnpnNwnJ2f2elso8tUzWFt087FX47wNvTICeUm5wX4+5FjtNl9IKk+DzecO44/vF3RKcSjXQ8RfgAAGJNpjbRymrpaO0yXoHTKGatWHnk9Qvj+bquo6xTqzns5NfGZYRfGmYj3DC9n+OcAXDjZGokUeUV1tANhqI/IPhu7+fzxPLfdGmqsbejDwWCEu2OYhnw8xRoq15sLkiIPNk4TUvhVr86Zh9M3sWp9OW9/WAvAtoo6Vt5UDMC/PrODfTXHmDcll08U5rD85Q9Co1zCx847N1MZkeYjJ5ASulL37Q9P/Htz1h1tamXnwQYumJxDbVMrTS3t7Klu4tKpeYzJSuP9g/XsrGrgW1eeyU/+tJvcQErUg85H9S2hsFswNY83PqihI2gIpPo4b8IoNpbX0h40XHRGLmOy0lhTWkkwaDWfTcjJYM6k0aGO+LwRqZ3eb+b4bHICKby6q5oFU/OoCSvngql5rN9d0ymYnQOGc4Bz2rxHpHqpbWrrdMADoh5MnAPn8fYgxeOz2W4PCgC6tLF77SNmis/D9bPG8dQ7VaHZUHWUy3A0kG3ukTTgXaO7MdcQn+mSeztML/LgFXnQuWLaGA7Vt3C0qTU0Zn7a2BND98JHUjnj4wGe317Fn947xBXTxgDW8MSH1+3limljmJQXYGN5LRvLj3Bd8dhO49lXrS+n4kgT/1N6AAQ+e+441pRWhg4i187I54nNldx4XgEHPramnHh43V5mjs+mtqkVAd6trGfeFGu0ijNT6ek51mit8APn3uomnnznALmBFMZkpdqd+nDjeQX8de8RKsImuOvvPEYa6MlkMGvuDg141Q99vdCnp+378n6RV1YXFVhj36Pd1rE/B8fIDn7nc2KNZe8PDfRklYhwD6dBr9Sg00AfLjTglUp6GujDVaIDHjTklYozDXQ1NMI9nAa9UidFA11FpyGvlOtooKveGWoBH0kDXykNdHWShnrAh9OwV8OEBrqKLw16pRKmX4EuImnAG0Aq4APWGmO+G7HNrcBPgAP2opXGmF90974a6EnETQEfSQNfuUx/A12AgDGmUUT8wFvAXcaYv4ZtcytQYoy5o7eF0kBPcm4OeYeGvRqCugt0X08vNlbiO3fs9ds/iWmnUe5RtCh6ELop6JuPwFO3Wz/d0eBXQ0Sv2tBFxAtsASYD/2mMuTti/a3AD4FqYDfwdWNMl/thicgSYAnAaaeddu7+/fv7W36VDMrWwKsPQF0Fg3fvnUGmoa/iJG6doiIyEnga+KoxZkfY8hyg0RhzXESWAouMMZd2917a5KJ6zU21+njQ8FfdiOsoFxH5LtBkjHkwxnovcMQYk93d+2igq7gYbmEfScN/2Olvp2ge0GaM+VhE0oGXgR8bY14I2ybfGHPQfvxp4G5jzJzu3lcDXQ2o4R70sYgHTBCyx8OC+/RA4EL9DfQi4LeAF+u2fWuMMQ+IyANAqTHmORH5IXAt0A4cAZYZY3Z1974a6CqhNPB7R88Ahhy9sEipvtCwjy89KMSVBrpSA0GDf/BoU1GIBrpSiaKhPzS5+KxBA10pN9DwTx7+APhSofkoZBfE9axCA12pZKPh734neZbQr0v/lVJDUKypFWIZDlfjuk3zEXj2K9bjONXeNdCVGg5O5gCgZwADr6PVOtBqoCulBkxfDwDR6EGhd+oq4/ZWGuhKqYERr4NCsjcVZRfE7a000JVSQ1c8DgrhhtpZgzfFGgETJxroSqnhI94HiEh9OWAMwFh4DXSllIqXgT5g9MCTsE9WSikVVxroSimVJDTQlVIqSWigK6VUktBAV0qpJJGwyblEpBrYfxIvzQVq4lycRNF9GZp0X4Ym3RfL6caYvGgrEhboJ0tESmPNNOY2ui9Dk+7L0KT70jNtclFKqSShga6UUknCjYH+SKILEEe6L0OT7svQpPvSA9e1oSullIrOjTV0pZRSUWigK6VUknBVoIvIlSLyNxHZIyL3JLo8fSUi+0TkXRHZJiKl9rLRIvJnEfnA/j0q0eWMRkR+JSKHRWRH2LKoZRfLCvt7KhORWYkreVcx9uV+ETlgfzfbROSqsHX32vvyNxG5IjGl7kpExovI6yLyvoi8JyJ32ctd9710sy9u/F7SRORtEdlu78v37OUTRWST/b08ISIp9vJU+/kee/2Ek/5wY4wrfgAvUA5MAlKA7cDZiS5XH/dhH5AbsezfgXvsx/cAP050OWOU/WJgFrCjp7IDVwF/xLrFzBxgU6LL34t9uR/4ZpRtz7b/raUCE+1/g95E74Ndtnxglv04E9htl9d130s3++LG70WAEfZjP7DJ/nuvARbby1cBy+zHXwZW2Y8XA0+c7Ge7qYY+G9hjjNlrjGkFHgc+leAyxcOngN/aj38LXJfAssRkjHkDiJy1P1bZPwX8zlj+CowUkfzBKWnPYuxLLJ8CHjfGHDfGfAjswfq3mHDGmIPGmHfsxw3A+8A4XPi9dLMvsQzl78UYYxrtp377xwCXAmvt5ZHfi/N9rQUWiIiczGe7KdDHARVhzyvp/gsfigzwsohsEZEl9rIxxpiDYP2jBk5JWOn6LlbZ3fpd3WE3RfwqrOnLFftin6YXY9UGXf29ROwLuPB7ERGviGwDDgN/xjqD+NgY025vEl7e0L7Y6+uAnJP5XDcFerQjltvGXF5gjJkFLAS+IiIXJ7pAA8SN39XDQCEwEzgI/D97+ZDfFxEZATwJfM0YU9/dplGWDfV9ceX3YozpMMbMBAqwzhzOiraZ/Ttu++KmQK8Exoc9LwCqElSWk2KMqbJ/HwaexvqiDzmnvfbvw4krYZ/FKrvrvitjzCH7P2EQeJQTp+9Del9ExI8VgI8ZY56yF7vye4m2L279XhzGmI+BdVht6CNFxLntZ3h5Q/tir8+m902Cnbgp0DcDZ9g9xSlYnQfPJbhMvSYiARHJdB4DlwM7sPbhH+3N/hF4NjElPCmxyv4c8Hl7VMUcoM5pAhiqItqSP4313YC1L4vtkQgTgTOAtwe7fNHY7ay/BN43xiwPW+W67yXWvrj0e8kTkZH243TgMqw+gdeBG+zNIr8X5/u6AXjN2D2kfZboHuE+9h5fhdX7XQ58J9Hl6WPZJ2H1ym8H3nPKj9VW9irwgf17dKLLGqP8f8A65W3DqlF8MVbZsU4h/9P+nt4FShJd/l7sy+/tspbZ/8Hyw7b/jr0vfwMWJrr8YeW6EOvUvAzYZv9c5cbvpZt9ceP3UgRstcu8A7jPXj4J66CzB/gfINVenmY/32Ovn3Syn62X/iulVJJwU5OLUkqpbmigK6VUktBAV0qpJKGBrpRSSUIDXSmlkoQGulJKJQkNdKWUShL/H/WOeUGZUEFuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "smallest val loss: (277, 3.751777)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'lmbda_l2': 0.2,\n",
       " 'd_out_rate': 0.1,\n",
       " 'num_layers': 4,\n",
       " 'd_model': 50,\n",
       " 'dff': 512,\n",
       " 'num_heads': 5,\n",
       " 'init_lr': 0.001}"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist(history)\n",
    "params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you can use more data or more regularization to avoid the overfitting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a tenminute video traces us presidential elections in order to provide a historical insight into the event the video discusses the elections held less than a year after the assassination of the th us president john f kennedy further detailing the history it describes the elections wherein barack obama was voted as the th us president\n",
      "\n",
      "video explains the us elections\n"
     ]
    }
   ],
   "source": [
    "print(clean_words(longreview_val['long'][i1]))\n",
    "print()\n",
    "print(summarize(clean_words(longreview_val['long'][i1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "brazilian police on wednesday arrested the head of the european olympic committees patrick hickey in rio de janeiro over illegal sales of olympic tickets police said hickey and at least six others are accused of illegally passing on tickets for the games to be sold on at extortionate prices hickey was taken to hospital after his arrest\n",
      "\n",
      " brazil police arrests rio olympics officials\n"
     ]
    }
   ],
   "source": [
    "print(clean_words(longreview_val['long'][i2]))\n",
    "print()\n",
    "print(summarize(clean_words(longreview_val['long'][i2])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}