emotion_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8Bro2KJwlvjk"
   },
   "source": [
    "<h1><center> Customised BERT Family Classsifiers for Emotion Analysis</center></h1>  \n",
    "<h2><center>BERT, DistilBERT, RoBERTa</center></h2>\n",
    "\n",
    "<h3><center>INM706 - Deep Learning for Sequence Analysis </center></h3>\n",
    "\n",
    "<center>A. Ascencio-Cabral<center>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XWFCjRWdFh7n",
    "outputId": "233f3662-ca06-4b45-cbf3-71b678113ba2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive/\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "_iHmBIs161kP"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install transformers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "85ozZmWrxGlV"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install torchmetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "l4HPn61epgeE"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "H914ZBwE3qv5"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install emot\n",
    "!pip install scikit-posthocs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qxoKh42Xyk9V"
   },
   "source": [
    "<h2> 1. Libraries </h2> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "_YiuzXhLFo0D"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import gensim.downloader as api\n",
    "import time\n",
    "import emot\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader, SequentialSampler\n",
    "from transformers import BertModel, DistilBertModel,  RobertaModel\n",
    "from transformers import BertTokenizerFast, DistilBertTokenizerFast, RobertaTokenizerFast\n",
    "from transformers import BertConfig, DistilBertConfig, RobertaConfig\n",
    "from transformers import get_linear_schedule_with_warmup\n",
    "from collections import defaultdict\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import class_weight\n",
    "from torch.optim.lr_scheduler import OneCycleLR\n",
    "from datetime import date\n",
    "from tqdm import tqdm\n",
    "from natsort import natsorted\n",
    "%matplotlib inline\n",
    "seed_= 1000\n",
    "torch.manual_seed(seed_)\n",
    "np.random.seed(seed_)\n",
    "\n",
    "plt.style.use('ggplot')\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "bZh9EDtg_I80"
   },
   "outputs": [],
   "source": [
    "os.chdir('/content/drive/MyDrive/INM706')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "tKV0LJY6_pZ1"
   },
   "outputs": [],
   "source": [
    "from evaluation_metrics import Metrics, train_metrics\n",
    "from comparison import save_file, get_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WRVwAtBvGH2o"
   },
   "source": [
    "<h2> 2. Emotion Dataset </h2> \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tdTvyL-gOw6B"
   },
   "source": [
    "\n",
    "<h3 id=\"reviewing_data\"> 2.1 About the dataset</h3>\n",
    "\n",
    "**`emotions.csv`:** \n",
    "The dataset comprises emotional content (tweets), tweet id, author and a single categorical label per content id. It has in total 40,000 tweets of emotional content. A subset of this data was used in a project for Microsoft's Cortana Intelligence Gallery. \n",
    "The dataset was added to DataWorld on July 15, 2016 by CrowdFlower and can be sourced from:\n",
    "https://www.crowdflower.com/data-for-everyone/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jIKc6ZovGmNp"
   },
   "source": [
    "<h3> 2.2 Downloading the dataset </h3>\n",
    "\n",
    "\n",
    "To download the data from dataworld we will use !wget\n",
    "https://query.data.world/s/m3dkicscou2wd5p2d2ejd7ivfkipsg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mypXbm3WF0b6"
   },
   "outputs": [],
   "source": [
    "# !wget -O emotions.csv https://query.data.world/s/m3dkicscou2wd5p2d2ejd7ivfkipsg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yrlVwvgWeGKy"
   },
   "source": [
    "<h3> 2.3 Reading the data from the CSV file </h3>  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "JlJGWDIUGD8A",
    "outputId": "a545986c-bec4-4ad9-e5a3-8e6b817ad21f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-92a1aa1a-d8eb-4eb1-8249-f42cfc640c78\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>author</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1956967341</td>\n",
       "      <td>empty</td>\n",
       "      <td>xoshayzers</td>\n",
       "      <td>@tiffanylue i know  i was listenin to bad habi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1956967666</td>\n",
       "      <td>sadness</td>\n",
       "      <td>wannamama</td>\n",
       "      <td>Layin n bed with a headache  ughhhh...waitin o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1956967696</td>\n",
       "      <td>sadness</td>\n",
       "      <td>coolfunky</td>\n",
       "      <td>Funeral ceremony...gloomy friday...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1956967789</td>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>czareaquino</td>\n",
       "      <td>wants to hang out with friends SOON!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1956968416</td>\n",
       "      <td>neutral</td>\n",
       "      <td>xkilljoyx</td>\n",
       "      <td>@dannycastillo We want to trade with someone w...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-92a1aa1a-d8eb-4eb1-8249-f42cfc640c78')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-92a1aa1a-d8eb-4eb1-8249-f42cfc640c78 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-92a1aa1a-d8eb-4eb1-8249-f42cfc640c78');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "     tweet_id   sentiment       author  \\\n",
       "0  1956967341       empty   xoshayzers   \n",
       "1  1956967666     sadness    wannamama   \n",
       "2  1956967696     sadness    coolfunky   \n",
       "3  1956967789  enthusiasm  czareaquino   \n",
       "4  1956968416     neutral    xkilljoyx   \n",
       "\n",
       "                                             content  \n",
       "0  @tiffanylue i know  i was listenin to bad habi...  \n",
       "1  Layin n bed with a headache  ughhhh...waitin o...  \n",
       "2                Funeral ceremony...gloomy friday...  \n",
       "3               wants to hang out with friends SOON!  \n",
       "4  @dannycastillo We want to trade with someone w...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./emotions.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "way12JBKrJe5"
   },
   "source": [
    "<h3> 2.4 Verifying that there are no empty cells </h3> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "B5ugW5dXEvtr",
    "outputId": "c4dcac7b-d672-44c4-a8fc-55f4fb635bca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 40000 entries, 0 to 39999\n",
      "Data columns (total 4 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   tweet_id   40000 non-null  int64 \n",
      " 1   sentiment  40000 non-null  object\n",
      " 2   author     40000 non-null  object\n",
      " 3   content    40000 non-null  object\n",
      "dtypes: int64(1), object(3)\n",
      "memory usage: 1.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OfXkkE3ZF9xG",
    "outputId": "10d097ad-92a3-48da-bf1a-3d1cdf2638ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num of instances: 40000\n"
     ]
    }
   ],
   "source": [
    "#review the sentiment instances\n",
    "print('Num of instances: {}'.format(len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 175
    },
    "id": "ZVATlCWumr5h",
    "outputId": "4ac609e6-1667-4cb5-9ada-011702b37692"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-dcace6de-10ce-4e40-a5cf-46a867f43577\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>40000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>8638</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dcace6de-10ce-4e40-a5cf-46a867f43577')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-dcace6de-10ce-4e40-a5cf-46a867f43577 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-dcace6de-10ce-4e40-a5cf-46a867f43577');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "       sentiment\n",
       "count      40000\n",
       "unique        13\n",
       "top      neutral\n",
       "freq        8638"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['sentiment']].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KMzTaRbseGKz"
   },
   "source": [
    "<h3> 2.5 Data visualisation </h3> "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AZpFn0d_S90c"
   },
   "source": [
    "Let's visualise the emotions distribution in the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 389
    },
    "id": "8zIhexsQjDCO",
    "outputId": "65e04112-8a82-4896-ca44-3325d4f3c82b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Emotion')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAFiCAYAAADLMs4qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd1gU5/o38O8CAgKhhSKKWIKA3YA1YgmCJopRY8tRLIiiiUTTo+EYu5Bi/KkJGhS7MWosENSgcCygRo0NFAE7IAIWFkTKCsz7B4d5WUEFmVmE8/1cl9flzsw+9zML7D0zT1MIgiCAiIhIYlq1XQEiIqqfmGCIiEgWTDBERCQLJhgiIpIFEwwREcmCCYaIiGTBBEP0Xxs2bICOjk5tV+OZFAoFtmzZ8szXUurbty8mT578zNdSa968ORYtWiRb+VQ7mGCoVkycOBEKhaLCPyMjI9ljp6amQqFQ4MiRI2rbR48ejTt37sgeXyp3797FiBEjqnTsli1boFAoqlz27t278dNPP71s1Z5p8uTJ6Nu3b4XtZ86cwaeffip5PKpdr+7lGtV7vXr1wo4dO9S2aWnV3jVPw4YN0bBhw1qLX12NGjWSvEyVSgVdXV2Ym5tLXvbzWFpaajQeaQbvYKjW6OrqolGjRmr/rKysxP19+/aFj48P/v3vf8PKygqmpqbw9/dHSUkJFixYAGtra1haWsLf31+t3EePHmHq1KmwtLSEnp4eOnfujIMHD4r7mzZtCgB4++23oVAo0Lx5cwCVPyLbv38/XFxcoKenBysrK3z00Ud4/PixuH/ixIlwd3dHcHAwmjVrBmNjY7z33nvIyMgQj0lNTcXw4cNhYWEBfX19tGzZEj/88MNzP5vDhw+jQ4cO0NfXR4cOHXD48OEKxzz9iGzt2rVo3bo19PX1YW5ujt69eyM1NRVHjhzBuHHjxPcoFApMnDhR7TOeM2cObGxsYGdnJ25/+pFYSUkJZs2aBQsLCxgbG8PX1xcFBQVqP6+n37No0SLx8503bx5CQkJw9OhRsR4bNmwAUPER2Yt+hrdu3YJCocCOHTvg6ekJAwMDtGzZUiyPXg1MMPRK++OPP/DkyRPExMTgp59+wpIlSzBo0CDk5uYiOjoaP/74I5YsWYIDBw6I75k0aRIiIiKwZcsWXLhwAT179oSnpycSEhIAAOfOnQMA7Nq1C3fv3sWZM2cqjR0bG4v33nsPvXv3xsWLF7Fx40aEh4dj2rRpasedOXMGhw8fxr59+xAREYG4uDh88cUX4v6PPvoI2dnZiIyMREJCAkJCQmBra/vMc05LS4OnpydcXFxw7tw5LF26FDNnznzu53T27FlMmzYNs2fPRmJiIo4ePYrx48cDAN566y38/PPPAEofq929exfLly8X37tjxw7cu3cPUVFROHTo0DNj/PHHH3jw4AGio6OxdetW7N27F7Nnz35uvcr74osvMGbMGPTo0UOsx+jRoys99kU/wzKzZs3C+PHjERsbiw8++ACTJ09GUlJSletEMhOIasGECRMEbW1twdDQUO2fp6eneEyfPn2Ejh07qr2vTZs2Qrt27dS2dejQQfj8888FQRCEq1evCgCEffv2qR3z5ptvCt7e3oIgCEJKSooAQDh8+LDaMevXrxe0tbXF115eXkKXLl3Ujtm7d6+gUCiEW7duiedhaWkpFBQUiMcEBgYKjRo1Uqvf3Llzq/KxCIIgCP7+/oKdnZ3w5MkTcduff/4pABA2b94sbiv/evfu3YKxsbGQnZ1daZmbN28WKvtz79Onj9CqVSuhuLi4wnYfHx+1182aNROKiorEbb/++qugp6cn5ObmVvoeQRCEhQsXCs2aNRNf+/j4CH369KlQj2bNmgkLFy4UBKFqP8ObN28KAISlS5eK+4uKigQjIyNh9erVlX4GpHlsg6Fa061bN2zcuFFtm4GBgdrrjh07qr0ue5T29LbMzEwAQHx8PACgd+/easf07t0bJ0+erFb9Ll++DDc3N7Vtffr0gSAIiI+PR7NmzQAATk5O0NPTE49p3Lix2iOyTz75BFOnTsWBAwfQt29fDBo0qEL9youPj0fXrl3VHte5uro+t64eHh5o2bIlWrRoAQ8PD7i5ueH999+HhYXFC8/TxcWlSm1fXbt2hba2tvi6Z8+eKCwsxPXr19GhQ4cXvr+qqvMz7NSpk/h/bW1tWFlZqX32VLv4iIxqTcOGDWFvb6/2r3HjxmrHNGjQQO21QqGodFtJSYns9X0WXV1dtdcKhQJCuUnKvb29cfv2bUybNg13797Fu+++Cy8vL0nrYGRkhH/++Qd79uyBg4MDVq9eDXt7e5w9e/aF7zU0NJSkDlpaWmrnDQBPnjyRpOxnqeyzr83fBVLHBEP1Stu2bQEAx44dU9t+7NgxtGvXDsD//1IqLi5+YVlPl1PWQF0Wp6psbGzg7e2NTZs2ISQkBFu3bkVOTk6lx7Zp0wanT59Wq9/x48dfGENbWxu9e/fGggULcPbsWdjY2OC3334DUPVzfp4zZ86ovf/EiRPQ09PDG2+8AQCwsrJCWlqa2nvK2rvK6OrqVulzB57/M6S6gQmGao1KpUJ6enqFf09fBVfHG2+8gZEjR+Kjjz5CREQEEhISMHPmTFy6dAlffvklAMDCwgJGRkY4ePAg0tPTkZWVVWlZX375Jc6dO4dPP/0UCQkJ+Ouvv/Dxxx9j7NixYm+rqvDz88P+/ftx/fp1XL58Gbt370bTpk3x2muvVXr8hx9+iHv37sHX1xdXrlxBVFRUhZ5yTwsNDcWyZctw9uxZJCcnY+/evUhJSUGbNm0AAC1atAAAhIWF4d69e8jNza1y/cs8ePAA06dPx5UrV7Bv3z7MmTMHU6dOFe+A3N3dERkZiZ07d+LatWsIDAxEdHS0WhktWrRAQkICLl++jPv376OwsLBCnKr8DKluYIKhWhMdHQ0bG5sK/x48eFCjcteuXYsBAwbAy8sLHTt2xPHjxxEeHg4nJycApY9yfvnlF+zYsQO2trZ48803Ky2nQ4cOCAsLw7Fjx9CxY0eMGzcOgwYNwurVq6tVH0EQ8Mknn6Bdu3bo3bs3Hj9+jAMHDjxz4GOTJk3w559/4vTp0+jUqRNmzpz5wkGPZmZm+PPPP/HOO+/AwcEBX331Ff7973/Dx8cHANClSxfMnDkTU6dOhZWVFfz8/Kp1DgAwYsQIvPbaa3B1dcUHH3wAT09PBAYGivsnTJiA6dOnY/r06ejcuTNSUlIwY8YMtTJ8fHzQpUsXvPXWW7C0tMS2bdsqjfWinyHVDQqhJpeLREREz8A7GCIikgUTDBERyYIJhoiIZMEEQ0REsmCCISIiWXCqmHKeHiT2IhYWFrh//75MtamfcerTudS3OPXpXOpbnFf5XJ6efaM83sEQEZEsmGCIiEgWTDBERCQLJhgiIpIFEwwREcmCCYaIiGTBBENERLJggiEiIlkwwRARkSw4kr8KmqxpUu333JlyR4aaEBHVHbyDISIiWTDBEBGRLJhgiIhIFmyDeYVUt62H7TxE9CrjHQwREcmCCYaIiGTBBENERLJggiEiIlkwwRARkSyYYIiISBYa6aZcUlKCHTt2IDo6GkqlEqampujVqxdGjhwJbW1tAIAgCNi5cyeioqKQm5uLVq1awcfHB02bNhXLyc3Nxfr16/HPP/8AADp37oxJkybB0NBQPCY5ORkhISG4du0ajIyM4OHhgeHDh0OhUGjiVImI6L80cgezd+9eREREwNvbG8uWLYO3tzciIiKwZ88e8ZjQ0FCEh4fD29sbAQEBMDY2xqJFi5Cfny8es2LFCty8eRP+/v7w9/fHzZs3sXLlSnF/Xl4eFi5cCBMTEwQEBMDb2xthYWEIDw/XxGkSEVE5GkkwSUlJcHFxQefOnWFlZYXOnTvDxcUF165dA1B697J//34MHToU3bt3h52dHfz8/JCfn4+YmBgAQGpqKi5cuABfX184ODjAwcEBvr6+OHfuHNLS0gAAMTExUKlU8PPzg52dHbp3744hQ4YgPDwcgiBo4lSJiOi/NJJgnJyccPnyZdy5UzryPDU1FZcvX8abb74JAMjMzIRSqUSHDh3E9+jq6qJ169ZITEwEUJqk9PX14ejoKB7j6OgIPT09tWOcnJygq6srHtOxY0dkZWXh3r17sp8nERH9fxppgxkyZAjy8/Px2WefQUtLC8XFxXj//fcxYMAAAIBSqQQAmJqaqr3PxMQEWVlZ4jHGxsZqbSkKhQImJibi+5VKJczNzdXKKCtTqVTCyspKbV9kZCQiIyMBAIGBgbCwsJDqlCUtS1MxdHR0ZK+3JmIwzqsbg3Fe3RhyxNFIgjlx4gSOHTuGGTNmoGnTprh16xbWr18PKysruLm5aaIKlXJ3d4e7u7v4+v79+5KVLWVZmophYWEhe701EYNxXt0YjPPqxnjZOI0bN37mPo0kmC1btmDw4MHo2bMnAMDOzg737t3Dnj174ObmpnaXUT57Zmdnw8TEBEDpnUhOTg4EQRDvYgRBQHZ2tvh+U1NTZGdnq8V+1t0RERHJSyNtMIWFhdDSUg+lpaUlNrxbWVnB1NQUsbGx4n6VSoWEhASxzcXBwQEFBQVISkoSj0lKSkJhYaHaMQkJCVCpVOIxsbGxMDMzg6WlpWznR0REFWkkwbi4uGDv3r04d+4cMjMzcfr0aYSHh6Nr164ASttSBg4ciNDQUJw6dQrJyckICgqCvr4+XF1dAQC2trbo1KkTgoODkZSUhKSkJAQHB8PZ2Vm8RXN1dYWuri6CgoKQnJyMU6dOITQ0FJ6enhwHQ0SkYRp5RDZp0iRs374da9euRXZ2NszMzNCvXz+MGDFCPGbIkCFQqVQICQnB48ePYW9vD39/fzRs2FA8ZsaMGVi3bh0WL14MoDRx+fj4iPsNDAwwZ84chISEYPbs2TA0NISnpyc8PT01cZpERFSORhJMw4YNMXHiREycOPGZxygUCowaNQqjRo165jFGRkaYMWPGc2PZ2dlh/vz5L1tVIiKSCOciIyIiWTDBEBGRLJhgiIhIFkwwREQkCyYYIiKSBRMMERHJggmGiIhkwQRDRESyYIIhIiJZMMEQEZEsmGCIiEgWTDBERCQLJhgiIpIFEwwREcmCCYaIiGTBBENERLJggiEiIlkwwRARkSyYYIiISBZMMEREJAsmGCIikgUTDBERyYIJhoiIZMEEQ0REsmCCISIiWTDBEBGRLJhgiIhIFkwwREQkCyYYIiKSBRMMERHJggmGiIhkwQRDRESyYIIhIiJZMMEQEZEsmGCIiEgWTDBERCQLJhgiIpKFjqYCZWVlYevWrTh//jwKCgpgZWWFKVOmoE2bNgAAQRCwc+dOREVFITc3F61atYKPjw+aNm0qlpGbm4v169fjn3/+AQB07twZkyZNgqGhoXhMcnIyQkJCcO3aNRgZGcHDwwPDhw+HQqHQ1KkSERE0dAfz+PFjzJkzBwAwe/ZsLFu2DJMmTYKxsbF4TGhoKMLDw+Ht7Y2AgAAYGxtj0aJFyM/PF49ZsWIFbt68CX9/f/j7++PmzZtYuXKluD8vLw8LFy6EiYkJAgIC4O3tjbCwMISHh2viNImIqByNJJjQ0FCYmZnBz88P9vb2sLKyQvv27WFrawug9O5l//79GDp0KLp37w47Ozv4+fkhPz8fMTExAIDU1FRcuHABvr6+cHBwgIODA3x9fXHu3DmkpaUBAGJiYqBSqeDn5wc7Ozt0794dQ4YMQXh4OARB0MSpEhHRf2kkwZw5cwb29vZYtmwZJk+ejC+//BJ//fWX+KWfmZkJpVKJDh06iO/R1dVF69atkZiYCABISkqCvr4+HB0dxWMcHR2hp6endoyTkxN0dXXFYzp27IisrCzcu3dPE6dKRET/pZE2mMzMTBw8eBCDBg3C0KFDcevWLaxbtw4A8M4770CpVAIATE1N1d5nYmKCrKwsAIBSqYSxsbFaW4pCoYCJiYn4fqVSCXNzc7UyyspUKpWwsrJS2xcZGYnIyEgAQGBgICwsLKQ6ZUnL0lQMHR0d2eutiRiM8+rGYJxXN4YccTSSYEpKSvDGG29gzJgxAIAWLVrg7t27iIiIwDvvvKOJKlTK3d0d7u7u4uv79+9LVraUZWkqhoWFhez11kQMxnl1YzDOqxvjZeM0btz4mfs08ojMzMxMbG8pY2trK55I+buM8rKzs2FiYiIek5OTo9aWIggCsrOzxfebmpoiOztbrYxn3R0REZG8NJJgHB0dxYb4MmlpaeKtmJWVFUxNTREbGyvuV6lUSEhIENtcHBwcUFBQgKSkJPGYpKQkFBYWqh2TkJAAlUolHhMbGwszMzNYWlrKdn5ERFSRRhLMoEGDcPXqVezevRvp6ek4efIkDhw4gAEDBgAobUsZOHAgQkNDcerUKSQnJyMoKAj6+vpwdXUFUHrH06lTJwQHByMpKQlJSUkIDg6Gs7OzeIvm6uoKXV1dBAUFITk5GadOnUJoaCg8PT05DoaISMM00gZjb2+PL7/8Etu2bcOuXbtgYWGB0aNHiwkGAIYMGQKVSoWQkBA8fvwY9vb28Pf3R8OGDcVjZsyYgXXr1mHx4sUAABcXF/j4+Ij7DQwMMGfOHISEhGD27NkwNDSEp6cnPD09NXGaRERUjsZG8js7O8PZ2fmZ+xUKBUaNGoVRo0Y98xgjIyPMmDHjuXHs7Owwf/78l64nERFJg3ORERGRLJhgiIhIFkwwREQkCyYYIiKSBRMMERHJggmGiIhkwQRDRESyYIIhIiJZMMEQEZEsmGCIiEgWL51gLl26hPj4eCnrQkRE9UiVE8zcuXORkJAAANi7dy+WL1+O5cuXY/fu3bJVjoiI6q4qT3aZkpICBwcHAEBUVBTmzp0LfX19zJkzB++//75sFSRpNVnTpNrvuTPljgw1IaL6rsoJpmwlyfT0dAAQV6h8/PixDNWiuo6JjIiqnGAcHR2xbt06ZGVloUuXLgBKk81rr70mW+WIiKjuqnIbzPTp02FgYIBmzZqJa7akpaVh4MCBslWOiIjqrirfwVy6dAljxoxR2+bs7Iy///5b8koREVHdV+U7mNWrV1e6/ddff5WsMkREVH+88A4mIyMDAFBSUoLMzEyxsb9sn66urny1IyKiOuuFCWbGjBni/z/++GO1faamphg5cqT0tSIiojrvhQlm+/btAEoHWs6fP1/2ChERUf1Q5TYYJhciIqqOKvciy8zMxLZt23Dr1i0UFBSo7Vu1apXkFSMiorqtyglm+fLlsLa2xvjx46GnpydnnYiIqB6ocoJJTU3FwoULoaXFGf6JiOjFqpwtWrdujVu3bslYFSIiqk+qfAdjaWmJxYsXo2vXrjA1NVXbN3r0aMkrRkREdVuVE0xhYSFcXFxQXFyMBw8eyFknIiKqB6qcYD766CM560FERPVMlRNM2ZQxlbG2tpakMkREVH9UOcGUnzLmaWWj/YmIiMpUOcE8nUSUSiV27tyJ1q1bS14pIiKq+156UIupqSkmTpyI3377Tcr6EBFRPVGjUZNpaWkoLCyUqi5ERFSPVPkR2bfffguFQiG+LiwsREpKCkaMGCFLxYiIqG6rcoJxc3NTe62vr49mzZrBxsZG8koREVHdV+UE07dvXxmrQURE9U2VE0xRURF2796NY8eOISsrC2ZmZujduzfef/996OhUuRgAwJ49e7Bt2zYMGDAAPj4+AABBELBz505ERUUhNzcXrVq1go+PD5o2bSq+Lzc3F+vXr8c///wDAOjcuTMmTZoEQ0ND8Zjk5GSEhITg2rVrMDIygoeHB4YPH672eI+IiORX5Ub+LVu2IC4uDlOmTMEPP/yAKVOm4NKlS9iyZUu1AiYlJSEyMhLNmjVT2x4aGorw8HB4e3sjICAAxsbGWLRoEfLz88VjVqxYgZs3b8Lf3x/+/v64efMmVq5cKe7Py8vDwoULYWJigoCAAHh7eyMsLAzh4eHVqiMREdVclRPM33//ja+++godO3ZE48aN0bFjR3zxxRc4efJklYPl5eVh5cqV+PDDD9XuOgRBwP79+zF06FB0794ddnZ28PPzQ35+PmJiYgCULhdw4cIF+Pr6wsHBAQ4ODvD19cW5c+eQlpYGAIiJiYFKpYKfnx/s7OzQvXt3DBkyBOHh4RAEocr1JCKimqtygpHiC/rXX39Ft27d0K5dO7XtmZmZUCqV6NChg7hNV1cXrVu3RmJiIoDSOx99fX04OjqKxzg6OkJPT0/tGCcnJ+jq6orHdOzYEVlZWbh3716N609ERFVX5caTHj164LvvvsOIESNgYWGB+/fvY9euXejevXuV3h8ZGYn09HR8/PHHFfYplUoAqLAMgImJCbKyssRjjI2N1dpSFAoFTExMxPcrlUqYm5urlVFWplKphJWVVYU6RUZGAgACAwNhYWFRpXOpCinLqs0YdTWOjo6ORupdn+LUp3Opb3Hq6rlUOcF4eXlh165dCAkJQVZWFszNzdGzZ08MHz78he9NS0vDtm3bsHDhwmp3CJCTu7s73N3dxdf379+XrGwpy6rNGHU1TtlFkNzqU5z6dC71Lc6rfC6NGzd+5r4XftsnJCTgn3/+gZeXF0aPHq22uNiWLVtw48YNODg4PLeMpKQkPHr0CJ999pm4raSkBFeuXMGhQ4fw008/ASi9yyifPbOzs2FiYgKg9E4kJycHgiCIdzGCICA7O1u8SzE1NUV2drZa7GfdHRERkbxemGD27NmDAQMGVLqvXbt22L17N2bNmvXcMrp06YIff/xRbduqVavQqFEjDBs2DDY2NjA1NUVsbCzs7e0BACqVCgkJCfDy8gIAODg4oKCgAElJSWI7TFJSEgoLC8XXDg4O2Lp1K1QqldgOExsbCzMzM1haWr7oVImISEIvTDC3bt1Cp06dKt3Xvn17rFq16oVBDA0N1XqNAYCenh6MjIxgZ2cHABg4cCD27NmDJk2awMbGBrt374a+vj5cXV0BALa2tujUqROCg4MxdepUAEBwcDCcnZ3FWzRXV1fs3LkTQUFBeP/993H37l2EhoZixIgRHAdTTzVZ06Ta77kz5Y4MNSGip70wweTn56OoqEitZ1aZ4uJitXEqNTFkyBCoVCqEhITg8ePHsLe3h7+/Pxo2bCgeM2PGDKxbtw6LFy8GALi4uIgDNQHAwMAAc+bMQUhICGbPng1DQ0N4enrC09NTkjoSEVHVvTDBNGnSBBcvXkSXLl0q7Lt48SKaNKn+FSQAzJs3T+21QqHAqFGjMGrUqGe+x8jI6LkLnwGAnZ0d5s+f/1J1IiIi6bxwHMygQYMQHByMU6dOoaSkBEBpA/2pU6ewZs0aDBo0SPZKEhFR3fPCOxhXV1colUr88ssvePLkCYyNjZGTk4MGDRpg1KhRYhsJERFReVUalOLp6Qk3NzckJSUhNzcXRkZGcHBwgIGBgdz1IyKiOqrKox4NDAye2ZuMiIjoaTVaMpmIiOhZmGCIiEgWTDBERCQLJhgiIpIFEwwREcmCCYaIiGTBBENERLJggiEiIlkwwRARkSyYYIiISBZMMEREJAsmGCIikgUTDBERyYIJhoiIZMEEQ0REsmCCISIiWTDBEBGRLJhgiIhIFkwwREQkCyYYIiKSBRMMERHJggmGiIhkwQRDRESyYIIhIiJZMMEQEZEsmGCIiEgWTDBERCQLJhgiIpIFEwwREcmCCYaIiGTBBENERLJggiEiIlno1HYFiOqCJmuaVOv4O1PuyFQTorpDIwlmz549OH36NNLS0qCjo4NWrVphzJgxsLOzE48RBAE7d+5EVFQUcnNz0apVK/j4+KBp06biMbm5uVi/fj3++ecfAEDnzp0xadIkGBoaisckJycjJCQE165dg5GRETw8PDB8+HAoFApNnCoREf2XRh6RxcfHo3///li4cCHmzp0LbW1tLFy4ELm5ueIxoaGhCA8Ph7e3NwICAmBsbIxFixYhPz9fPGbFihW4efMm/P394e/vj5s3b2LlypXi/ry8PCxcuBAmJiYICAiAt7c3wsLCEB4eronTJCKicjSSYPz9/fH222/Dzs4OdnZ2+Pjjj5GTk4OEhAQApXcv+/fvx9ChQ9G9e3fY2dnBz88P+fn5iImJAQCkpqbiwoUL8PX1hYODAxwcHODr64tz584hLS0NABATEwOVSgU/Pz/Y2dmhe/fuGDJkCMLDwyEIgiZOlYiI/qtWGvnz8/MhCAKMjIwAAJmZmVAqlejQoYN4jK6uLlq3bo3ExEQAQFJSEvT19eHo6Cge4+joCD09PbVjnJycoKurKx7TsWNHZGVl4d69e5o4NSIi+q9aaeRfv349mjdvDgcHBwCAUqkEAJiamqodZ2JigqysLPEYY2NjtbYUhUIBExMT8f1KpRLm5uZqZZSVqVQqYWVlpbYvMjISkZGRAIDAwEBYWFhIdYqSllWbMRjn1Yiho6Mje701EYNxXt0YcsTReILZuHEjEhMTsWDBAmhp1W4vaXd3d7i7u4uv79+/L1nZUpZVmzEY59WIYWFhIXu9NRGDcV7dGC8bp3Hjxs/cp9Fv+A0bNuD48eP49ttvYW1tLW4vf5dRXnZ2NkxMTMRjcnJy1NpSBEFAdna2+H5TU1NkZ2erlfGsuyMiIpKXxhLM+vXrxeTSpIn6mAIrKyuYmpoiNjZW3KZSqZCQkCC2uTg4OKCgoABJSUniMUlJSSgsLFQ7JiEhASqVSjwmNjYWZmZmsLS0lPP0iIjoKRpJMGvXrsWRI0cwc+ZMGBkZQalUQqlUoqCgAEBpW8rAgQMRGhqKU6dOITk5GUFBQdDX14erqysAwNbWFp06dUJwcDCSkpKQlJSE4OBgODs7i7dorq6u0NXVRVBQEJKTk3Hq1CmEhobC09OT42CIiDRMI20wBw8eBAAsWLBAbfuIESMwatQoAMCQIUOgUqkQEhKCx48fw97eHv7+/mjYsKF4/IwZM7Bu3TosXrwYAODi4gIfHx9xv4GBAebMmYOQkBDMnj0bhoaG8PT0hKenp9ynSFRj1Z0tAOCMAfRq00iC2bFjxwuPUSgUGDVqlJhwKmNkZIQZM2Y8txw7OzvMnz+/2nUkIiJpcbJLIiKSBRMMERHJggmGiLIXUjwAACAASURBVIhkwQRDRESy4HowRP9j2FuNNIV3MEREJAsmGCIikgUTDBERyYIJhoiIZMFGfiKSBTsTEO9giIhIFkwwREQkCyYYIiKSBRMMERHJggmGiIhkwQRDRESyYIIhIiJZMMEQEZEsmGCIiEgWTDBERCQLJhgiIpIF5yIjojqtunOecb4zzeEdDBERyYJ3MEREL8CZoV8O72CIiEgWTDBERCQLJhgiIpIFEwwREcmCCYaIiGTBBENERLJggiEiIlkwwRARkSyYYIiISBZMMEREJAsmGCIikgUTDBERyYIJhoiIZFEvZ1OOiIhAWFgYlEolbG1tMXHiRLRu3bq2q0VE9Fz1bdbmepdgTpw4gQ0bNsDHxwdOTk44ePAglixZgmXLlsHCwqK2q0dEVOs0lcjq3SOy8PBw9OnTB+7u7rC1tcWkSZNgZmaGgwcP1nbViIj+p9SrBFNUVIQbN26gY8eOats7dOiAxMTEWqoVEdH/JoUgCEJtV0IqDx8+xLRp0zBv3jy0adNG3P7HH38gOjoay5cvVzs+MjISkZGRAIDAwECN1pWIqL6rV3cw1eXu7o7AwMCXTi6zZs2SuEb1P059Opf6Fqc+nUt9i1NXz6VeJRhjY2NoaWkhOztbbbtSqYSpqWkt1YqI6H9TvUowOjo6aNmyJWJjY9W2x8XFwdHRsZZqRUT0v0l73rx582q7ElJq2LAhduzYATMzM+jq6mLXrl2Ij4/HRx99BENDQ8njtWzZUvIy63uc+nQu9S1OfTqX+hanLp5LvWrkL1M20DIrKwtNmzbFhAkT1Br9iYhIfvUywRARUe2rV20wRET06mCCISIiWTDBvIKKi4truwpEVM8VFRXhm2++QVpammwxmGBeQb6+vti0aRNSU1Nruyo1VlJSgpKSEvG1UqlEVFQUEhISarFWNRMREYHPPvsMXl5eyMjIAADs3bsXJ06cqOWaVc3o0aPFsWJBQUHIz8+v5RpJ4/Tp02q/a/R8Ojo6yMzMlDeGrKXXA7m5uVU+1sjISJKY//rXv3DkyBHs27cP9vb26NevH9566y3o6+tLUn6Z+Ph4NGjQAK1atQIAHDlyBFFRUWjatCnGjx8vSbzAwEB06tQJAwcOREFBAWbNmoXCwkIUFBRg2rRp6NOnT41jlKdUKnHs2DFkZGRg9OjRMDY2RkJCAszNzWFlZVXj8vft24ewsDAMGTIEW7duFbebm5sjIiICb731Vo1jAKWzgsfFxSE7OxtP98P5+uuva1S2rq4uCgoKYGJigqNHj8LLywsNGzasUZkvcv78eURERCAjIwP+/v6wsLBAVFQUrKys0L59e0lirFy5Evr6+ujTpw/c3NzQuHFjScp92unTp3Hp0iXk5ORUSGifffaZpLGKioqQnJxcaSxnZ+cal9+nTx9ERUVh3LhxNS6rMkwwL+Dj41PlY7dv3y5JTHd3d7i7uyM1NRWHDx/G77//jg0bNqB79+5wc3ODk5OTJHE2bNiAkSNHAgDS0tIQHBwMNzc3JCQkYPPmzZgyZUqNY1y/fh1eXl4AgFOnTqFhw4b4+eefERMTgz///FPSBHPjxg0sWLAAVlZWSElJwXvvvQdjY2PExsbi7t27mDlzZo1jHDp0CFOnToWzszN+//13cXuLFi2QkpJS4/IBYPPmzdi/fz/atm0LMzMzKBQKScot4+joiB9++EEc77Bu3Tro6upWeuxHH31U43jR0dFYs2YN3NzcEBcXJz4CLikpQVhYmGQJJjg4GDExMThy5Aj+/PNPODg4wM3NDT169JDs4mzTpk04cOAAnJycYGJiAi0t+R4CxcbGYuXKlcjJyal0vxTfN4WFhYiOjkZsbCxatmwJPT09tf2TJk2qUflMMC8wd+7cWotta2uLcePGYezYsYiIiMCWLVtw9OhR2NjYYODAgXB3d6/RL3h6ejrs7OwAAH///Tc6dOiAyZMn4+rVq1i6dKkkCaagoAAGBgYAgIsXL6Jr167Q0dFBu3btEBISUuPyy9u8eTMGDhyIUaNGYfz48eL2Tp064ciRI5LEuHfvHpo2bVphu7a2NlQqlSQxjh07hpkzZ6J79+6SlPc0Pz8//Pnnn0hPTwdQepeuoyPfV0FYWBimTp2Knj174j//+Y+4vVWrVtixY4dkcRo2bAgPDw94eHggJSUFhw8fxm+//YYNGzagR48ecHNzg4ODQ41iHD16FJ999hm6dOkiUa2fLSQkBC4uLhg+fDhMTEwkv9AAgDt37ogXGnI8LmOCeYHaHKBZVFSEU6dO4fDhw7h06RKcnJzg5uaGhw8fijMUfPLJJy9dvkKhEG+7L126hK5duwIATE1N8ejRI0nOwcLCAomJiTAyMsLFixfFRwi5ubnPvGp+WTdu3MC0adMqbDc1Na0wP93Lsra2xs2bN2Fpaam2/fz587C1tZUkRklJCZo3by5JWZUxNTUVH4lMnz4dM2fOxGuvvSZbvLt371b6xa6vr4+8vDxZYjZt2hSDBg2Cnp4ewsLCcPLkSRw9ehQtWrTA1KlT0axZs5cqV1dXF02aVH+xrpeRlZWFYcOGVfhdk5LcF9BMMC9JqVSiqKhIbZtUK2beuHEDhw8fxvHjx6GtrY0+ffpg0qRJas+UXVxcajzzqb29PXbt2oUOHTrgypUr8PX1BVB6lW5mZlajsst4enri559/hr6+PiwsLMSlq+Pj48W7J6no6uri8ePHFbanpaXB2NhYkhiDBw9GSEgICgsLIQgCkpKScOzYMYSFheHDDz+UJIa7uzuOHTuGUaNGSVLe8/zyyy+yxzAzM8Pdu3crfFFeuXIFjRo1kjRWUVERTp8+LV6U2dvbY8qUKXjrrbeQm5uLbdu24f/+7/+wbNmylyp/6NChCA0Nha+vL7S1tSWt+9OcnZ2RmJgIa2trWeMAQE5ODjIyMtC8eXM0aNBAsnKZYKohLy8P69atw8mTJyskF0C6NpjZs2ejY8eO8PX1RZcuXSr9Rba0tETPnj1rFGfChAlYsWIFzpw5g/fff1/8Yz958mSNHyWU8fDwQMuWLfHgwQN06NBBfKTXqFEjjB49WpIYZTp37oydO3eKd0kKhQKZmZnYunUrunXrJkmMt99+G8XFxdi2bRtUKhV+/vlnmJmZwdvbW7IG/sePH+P48eOIi4uDnZ1dhZ9/TZ+LPy0iIgIRERHIzMzE0qVLYW1tjb1798LKykqSc3J3d8f69esxdepUAMCDBw9w5coVbNmyRWwDlMK6detw/PhxAEDv3r0xYcIEtbtKc3NzjB07VqzHy+jXrx/Onj2LadOmwcbGpsLPRso7Al9fX6xYsQI3btyo9PdAivbL/Px8rFq1CqdOnQIArFixAtbW1ggODoapqWmNL3I4VUw1rF69GtevX8fYsWOxdOlSfPjhh3j48CH279+P8ePHS/LMvLi4GIcOHcJbb70l2VV3dalUKmhpacn2XL6oqEiWsvPy8hAQEIDk5GQUFBSIj8YcHR0xe/ZsyXvh5eTkQBAEmJiYSFru/Pnzn7tfyi+xp3vF/fTTT7C2tsaxY8cQFRX1wrpU1bZt27Bv3z48efIEQGkX2cGDB+ODDz6QpHwAWLBgAfr164du3bo98/eruLgYiYmJL/3oe9WqVTh58iQ6depUabuIlMn/xIkT+OWXX1BUVARdXV21WAqFAhs3bqxxjLVr1+L27dvw8fHBnDlz8OOPP8La2hpnz57F77//jh9++KFmAQSqsqlTpwrx8fGCIAjC+PHjhbt37wqCIAjR0dHCggULJIszZswYISMjQ7LynqW4uFgoLi4WX2dlZQmRkZFCQkKCZDH27dsnnDx5UnwdFBQkjB49WpgxY4Zw584dyeKUFxcXJ4SGhgp79+4VLl68KGnZmvjMNGnmzJnC2bNnBUEQhHHjxgnp6emCIAhCcnKy4O3tLWmsgoIC4dq1a8LVq1eF/Px8ScsWBEG4fPmyUFRUVGF7UVGRcPnyZUlieHl5Sf479SzTpk0TNm3aJMtnVWbq1KnC1atXBUFQ//nfvXtXGDduXI3L50DLanj8+LH4HNnAwEBsCHdwcEBSUpJkcZo1ayb28JFTYGAg/vrrLwAQx6hs2bIF8+bNw9GjRyWJceDAAfFOLD4+HidPnsSMGTPQvHlzbNq0SZIYZW7dugUAaNeuHd577z0MGTIEHTp0kDSGJj4zTdJEr7iywZx6enp44403YG9vD319fRQUFCAoKEiSGEDpnV9l49by8vIkuxMzNjaGubm5JGW9SF5eHjw8PCS/8y7v8ePHlXbwKCgokKQLNttgqqFRo0bIyMiAhYUFmjRpguPHj8Pe3h6nT5+WbJAlAIwcORKbN2/GyJEj0bJlywq/YFLF0sQYlYcPH4oDHM+ePYvu3bvjrbfegp2dneQ9WL7++mvY2tqiV69e6NWrF15//XVJywc085l99913z91f04GW5WmiV9zRo0cxduzYCoM5VSoVjh07JslYmzKVdeV99OiRZF/So0aNwvbt2zF9+nRZv/gBoFu3boiLi5O8I0R5b7zxBv755x8MGjQIwP///A4dOiTJIo1MMNXQp08f3L59G23btsXQoUMRGBiIiIgIlJSUwNvbW7I4gYGBAIClS5dWul+qzgSaGKNiYGCAnJwcWFhYIDY2Fu+99x6A0mfwUl0hl1m2bBliYmLEwalOTk7o3bs3unfvLp5nTWniM3v6irKoqAi3b9/GgwcPxK7kUpGzV1xubq44C8Hjx4/VGqlLSkpw7tw5SdqvyifklStXqrW/lJSUICUlRbJOK2FhYbh37x6mTJkCCwuLCg3vP/74oyRxAMDKygq///47rly5Ajs7uwrtSp6enjWO8a9//QuLFy9GSkoKiouLER4ejpSUFFy7dk2Suz4mmGoo/wNt164d/u///g/Xr1+HjY2NpF1uNTW4UxNjVDp06IBff/0VLVq0QHp6Ot58800AQEpKiiRTt5TXuHFjjBo1CqNGjcLVq1cRHR2Nbdu2Yd26dXB2dpZkGg9NfGbPuqLftGmT5FO6yNkrrvwsGJ9++mmF/QqFQpKu2OUTsqGhodrPQUdHB05OTujXr1+N4wCQbfBrZQ4fPgx9fX0kJiYiMTFRbZ9CoZAkwTg6OmLRokUICwtDo0aNEBcXhxYtWmDx4sWSfKexF1kVFRUV4dtvv4Wfn59scxxpMg5Qehu8bt06cYzKd999By0tLezfvx9nzpyRJNHl5eXh999/x/3799G/f3906tQJALBjxw7o6Ojg/fffr3GM57l69SrWrFmD27dvS3Lnp4nP7FnS0tLw7bffYu3atbKUL3WvuPj4eAiCgAULFuDzzz9Xe7Sro6MDCwsLSdszdu7cicGDB8v+6IqqjncwVaSJmUc1GQfQzBgVAwODSrtuyjmIMDMzE9HR0YiJiUF6ejpat25d6Qj/l6HJcT1Pk3NadQCSd4sv6wr8888/4/XXX5d13i4A4pia9PR03LlzBwDQpEkTSdswyuYFK/uskpOTceLECdja2sLV1VWyOE8rKCgAAMmT5/3795+5T1dXt8a/E7yDqYbNmzcDgGwzj2o6jqbIPcNxmb/++gsxMTG4evUq7Ozs0KtXL7i6umqs149U1q1bV2FbVlYWLly4gLfffrvGYy2++OILzJs3D0ZGRvj888+fO8eVlG0KDx8+xP379ysMUpZqOqbc3FwEBQXh7Nmz4jkJggAXFxd8+OGHkkyHM3/+fPTq1Qtubm7IycnBzJkzYWZmhgcPHmDEiBEYPHhwjWOU99dffyE0NBQPHz4EALz++usYMmQIBgwYIEn5L7ooMjAwQN++feHl5fVSMxfwDqYa5J55VNNxAPlHcWtihuMyoaGh6NmzJ3x9fSVtEwsPD0f//v2hq6uL8PDw5x77ss/F4+Pj4ejoCG1t7QqzMisUChgbG2PChAl4++23X6r88rp16yZOB6KJNoWHDx9ixYoVuHLlSqX7peq0smrVKmRkZGD+/PniEhRXr17F2rVr8euvv+KLL76ocYyyQYlA6QSxjRo1QkBAAM6cOYMtW7ZImmB2796NvXv3YvDgweIM6leuXMFvv/2G/Px8DB06tMYxZs6ciS1btsDDw0PtM4uMjMTIkSORl5eHXbt2oWHDhi/11IEJphrknnlU03E0sbaJJmY4LhMUFCTLjLMHDhxAnz59oKuriwMHDjzzuJo0vM6fPx/BwcEwMTHBvXv3EBAQINsElOWnZ5FyqpZn2bBhA7S0tPDTTz9h9uzZ+Oabb5CdnY0dO3ZgwoQJksW5ePEivv32W7UeY05OTvD19cXChQsliaFSqcTHVHFxcXBxcQFQulzD8x43vYxDhw7B19dX7dFb+/btYWNjg23btkmSYA4ePIgJEyaoTaXUrl07NG7cGPv378f8+fNhbGyMnTt3MsHITVO9uzQVRxNrm2hihuMyCoUCSqUSERERSE1NhUKhgK2tLfr37w9TU9OXLrf8hJByTQ5pZGSEzMxMMcHUpyfXV65cwaxZs9CkSRPxbszJyQkNGjTA9u3bJRsMa2xsXOFuHyhtS5Bq7JiNjQ1OnTqFbt26qXW7z87OhqGhoSQxyuTk5MDe3r7Cdnt7e8n+dq5evVrp3GxNmzbF9evXAZQOJH/w4MFLlc8EUw1BQUHw9vau0FW0oKAA69atk3TAGFB6tZSeng6FQgFra2vJp7fXxChuTcxwXCYhIQFLliyBiYmJeBUbHR2Nffv2wd/fv8ZjIeTs4detWzfMmzdPTISzZs16ZqP4zz//XKNYL2p3KU+KNhiVSiXejRkZGSEnJweNGzeGra0tbt++XePyy4wYMQIbNmzAxx9/LLa7PXz4EJs3b8aIESMki7F8+XJs2rQJ7du3Fx8rXbx4ES1atJAkRhkbGxvExMRUqHtMTIxkv3+WlpaIjIys0N4bFRUlzg6fk5Pz0gmaCaYaNDUiuaioCNu2bcNff/0lNojq6Ojg3XffxQcffCDZRJGaGMWtiRmOy2zevBk9e/bElClTxC/nkpISrFmzBps2bcKiRYtqVL6cPfymTJmCzp074+7du9i0aRP69u0r2zLGmhzLAZSOT0pLS4OVlRWaNWuGgwcP4vXXX8dff/0laQeMffv24d69e5g+fbpagmnQoAGys7PVHm++bOLs1q0bgoKCkJWVpbamTPv27SX/fR45ciSWLVuGK1euiKPqExMTER8fX+m4opcxfvx4LF26FOfPn8cbb7wBoPSpQ3p6Oj7//HMApbNX9OjR46XKZ4KpAk2NSC6zdetWHD9+HFOmTBEb9xISEvDbb7+hpKRErS2jJjSxtsm4ceMQEBCAyZMno7CwEHPmzBFnOJZyJl2gdC6y6dOnq135a2lpwdPTE1999ZUkMeRaw1yhUIhrrN++fRuDBw+WLcFoot2lvIEDB0KpVAIovQNYsmQJjh8/Dh0dHUyfPl2yOJpKnKamphUeuZbdyUipW7duWLJkCcLDw3H27FkApd2ulyxZItndkrOzM5YvX46DBw+K3eBdXFzQv39/8Q6mJj3W2E25Cl7Ula9sRLJUgwanTJmCDz/8UPzCKXPu3DmsXr0awcHBksQBgMjISOzevVt8xmpmZoZRo0bBzc1NshhA6YqZN27cgCAIaNGiheSTUAKln9v06dPFwZxlzp8/j1WrVknyua1duxbR0dGwsrKSvYefJl2/fh0ZGRlwdnYWJ6Js0KCBLItqFRYW4s6dO7CwsKi1JSlqIi0tDX///XelXa6lfkxe1/EOpgrmzp2r0RHJeXl5lQ4Os7a2rrQ9oybc3d3h7u4u29omZdq1a4d27drJUnaZnj17YtWqVfDy8hIfKSQkJGDr1q01XpytjKZ6+GmKUqnEDz/8gGvXrgEoXXBKX18fmzZtQoMGDSSbY+/EiROIi4tDdnZ2hQ4MUk7eWUalUlWIU1kHgOo6d+4cli5diubNm+PGjRuwt7dHeno6ioqKxKcNNVGdnmhSraBbWFiIW7duVfqzqeljPyaYKig/ItnCwkKWrrDlNW/eHPv378fkyZPVtu/fv1+2tdrlvJK8evUq4uLikJOTg5KSErV9Ul7xe3l5QRAErFq1CsXFxQBKLwA8PDwwduxYSWJoqoefpmzcuBEmJiYICQlRu/ru3r071q9fL0mMzZs3Y//+/Wjbti3MzMxk+/u5d+8e1q9fj8uXL4sj38uTYrzN9u3bMWLECAwbNgzjx4+Hn58fzMzM8PPPP0syoWZ1HhlKcT6xsbFYvnx5pcscSBGDCaYaHj16JK4BU5myK9uaGjt2LAICAhAXF6c2+CkrKwvffPONJDEAiGuUX7p0qdKrFylWzAsLC8PWrVvRqFEjWb9cgNJk4u3tjTFjxiAjIwNA6V2fFFeuTyvr4QeUThMjdQ8/Tbl06RLmzJlToZdQo0aNJBvXcezYMcycOVP2NpKVK1fiyZMn8Pb2rlG39OdJS0sTx4dpa2ujsLAQurq6GDFiBAICAmo8AWVAQIBarK1bt8LDw0NMXklJSTh06JBkF0wbNmyAs7Mz/vWvf8ky4wUTTDXMnj37ufulGpHcpk0bseEtNTUVANCjRw/0799f0l+CVatW4datW+jXr59s06kcOHAA3t7eeOedd2Qp/0Vrp5QnxaOYJ0+eYOvWrTh06JBaDz93d3eMHTu2ziUalUpVaa/EnJwccbR/TZWUlMh2513ezZs3ERAQIFkPyMo0bNhQXPbZzMwM6enpsLOzQ3FxsSSPr8tfpG7cuBETJkxQS8zlB0FKMffZvXv38NVXX8n2988EUw1Pjz8oLi7GzZs3sXv3bowZM0ayOIsWLULbtm3x5ptvYuTIkbI0tAKlV6///ve/ZekBUyYvL0+col8Oco12f5Y1a9YgNjYW06ZNU7uqLJu+o6418jo5OeHIkSPi769CoUBJSQn27t0rWZuZu7s7jh07JusEp0Dpo+WyySjlYm9vj4SEBNja2sLZ2RmbN2/G7du3cfr0acnWnClz7dq1Sqc8srOzw40bNySJ4ejoiLS0NNkWNWOCqYanx4sApY8SDAwM8Mcff0j2RWpvb4/z58/jjz/+gLa2NhwcHNCmTRu0bdsW9vb2kiUcY2Nj2ac279mzJy5cuCDZ5HxP0/QX+t9//40vvvhCrRectbU1TExMsHTp0jqXYMaNG4e5c+fi+vXrePLkCTZt2oSUlBTk5eXVaHqV8hN2CoKAmJgYxMXFwc7OrsLvr1TtcL6+vli/fj3effddNG3atMKdmRSN4hMmTBDbd0aOHIn8/HycOnUKNjY2kk57A5QuOHbw4EFMnDhRbfvBgwcr/S56GR4eHti8eTOysrIq/dnU9LE/E4wErKysxPXgpVA2PkSlUiExMRGXL1/GhQsXsHPnTujq6krSNgKUrma3fft2+Pn5yZZoLCwssGPHDiQmJsq2Kp8m6enpVfo4wdzcvM49HisqKkJQUBC+/vprXLhwAQ0aNMCTJ0/Qo0cPDBgwAGZmZi9d9tNTDZU9IpNzyQFBEJCdnf3MQZRSPMK2trYW/6+np4cpU6bUuMxnmTBhAn744QdcuHBBfMpw7do1ZGZmSjJxJwD89NNPAPDMLvxs5Negp3taCIKArKws7Ny5U5bFwfLy8vDo0SPk5OQgOzsb2traknUkAIBdu3bh3r17mDx5MiwtLWVZ/jUqKkr2Vfk06d1338XOnTsxffp0MaGoVCr88ccfsrUzyaVsZgIjIyPJH1/VRm+7X375BcbGxvj6669hYmIie29PuXXq1AkrVqxARESEmJi7du0KDw8Pyboo13TaoRfhQMtqeNaAy9dffx2ffPKJZM9g165di8uXL+P+/fuwt7cXH4+1atVKsoZXoHQFwOfR9IjvuiAwMBBXrlyBlpaWOFVIcnIyiouLK6xrIsf4DqnVp7WHvLy88P3330t+sTdhwgSsXLkSxsbGGD9+/HMTl1RPFzSpuLgY165dqzBwVKFQoHfv3jUqm3cw1fD0VVnZzLCNGjWStCH+0KFDMDY2xpAhQ/Dmm2+iZcuWslyNMYFUn7GxcYXBZ1IumqZpmlx7SG729vbIzMyUPMGUn+BW059HcnIyDh06hMzMTEybNg1mZmY4ffo0LC0tJZku5s6dO/juu++QmZkJQRCgpaWFkpISaGtro0GDBkwwmiTVynsvsnz5csTHx+Py5cuIiopCQUEBHB0d0a5dO7Rp00bSx2RyCAoKqvKxdalRvKSkBO+99x4sLCzqzbrv9Wlmgv79+2Pjxo148OCBpA3Wffv2rfT/crt48SK+//57dOrUCXFxceIM5xkZGThy5Igk8+tt2LABLVu2xPfffw9fX198//33yMvLw9q1ayWZK5AJpprKrigyMjLw4YcfSn5FAZT2TGvUqJE4H9idO3cQGhqKrVu3oqSkpEYNb5q43X+6q2jZI6WyLpfJyckQBAGtW7d+qfJri0KhwJdffolly5bJ1q1T0+rTzATLly8HIF+DNfD/f7fLZr5ITk7GiRMnYGtrK8m4lPK2b9+O8ePHY8CAAWoT3LZt2/aFK6tW1fXr1zFv3jzo6+uLXdRbtmwJLy8vrFu3rsbtsEww1VD+iuLSpUuyXFEApVfKN27cwKVLlxAfH4/ExESoVCq0bNkSbdu2rVHZmrjdnzVrlvj/PXv2QFdXFx999JF41V9QUIDVq1dLuqyxJigUCjRu3Bg5OTn1JsHUJ3I3WAPAsmXL0KtXL7i5uSEnJwdz586FmZkZDhw4gKysLEmXTE5JSal06IORkdEzp3apLkEQxMeixsbGePjwIRo3bgxzc3NxpoqaYIKpBk1cUQClSeDJkydo0aIF2rZti4EDB8LJyUmSxzKavt0/cOAAvv32W7W66+vrY/jw4ViwYIFkM1BripeXFzZv3gwfHx80a9aszvdUqk8sLS1x/vx5REREICMjA/7+/rCwsEBUVBSsrKwkGTty+/Zt+Pj4ACgdE9WoYT8dTwAADz5JREFUUSMEBATgzJkz2LJli6QJxsjICA8fPqzQxnfjxg3JRt7b2dnh9u3bsLa2hr29PUJDQ6GlpYXIyEhJLqKYYKpBE1cUAPDpp59KllCq4tKlS+KUNLa2tpLOelxQUICHDx9WmL4jKytLslUzNWnZsmV48uQJvv76a7EhtLy62IuovoiOjsaaNWvg5uaGuLg4ccLTkpIShIWFoX379jWOoVKpxL/LuLg4uLi4AChdZlyqudvK9OzZE1u2bMGnn34KhUKB4uJixMfHY/PmzXj77bcliTFs2DAUFhYCKB1/FxgYiPnz5+O1116TZFEzJphq0MQVBYAK65nIJTMzE0uXLsXt27fFQXVlI3o///xztUFlL6tbt27iFPrlJ+7cunUrunbtWuPyNa0u9ar6XxMWFoapU6eiZ8+e+M9//iNub9WqFXbs2CFJDBsbG5w6dQrdunVDbGws3nvvPQBAdnY2DA0NJYlR5oMPPkBQUJDYEeazzz6DIAhwdXWV7M6//HeNtbU1li1bhtzcXBgaGkpyd84EUw2auKLQpFWrVkFfX19chgAoXY/il19+werVqyVpAJ4yZQo2bdqEoKAgsY+9trY23Nzc6uTYC032IqLquXv3bqVj0fT19ZGXlydJjBEjRmD58uXYtGkT2rdvL140Xbx4UbJOPkDpXVd6ejp8fX0xevRo3Lx5EyUlJWjRogVsbGwki1OZp2fWrgkmmGp43hXFsGHDarl21ZeUlITFixerjQq2sLDAhAkT4O/vL0kMXV1dTJ48GV5eXmpT6NfVbr4vehQq5R8nVY+ZmRnu3r1boa3lypUrknXK6NatG4KCgpCVlSUOtAWA9u3b13hxrvKe7rEoxdOE2sAEUw06OjqYMWOGxq8o5GJhYVFpO4hKpZJsKooy+vr6an+QdVVZA++zSLVkA1Wfu7s71q9fj6lTpwIAHjx4gCtXrmDLli2SDio2NTWtsN6M1DOS15cei5wqppo0vfSrnM6ePYs//vgD3t7esLe3B1A6md7GjRsxbNgwdO7cWZI4ly5dwvHjxytdw7yujcOIj49Xe11UVIRbt27h4MGD+OCDDyQfC0HVs23bNuzbt09cs0VHRweDBw+WZNAgoD5LdGWkbKM7f/48du/eXad7LDLBVMOLln6tS6PSAWD8+PF48uQJSkpKoKWlBQDi/5+eGfhle0cdOXIEa9asQZcuXXDmzBl07twZd+/eRWZmJnr16vXCO4K64u+//8Z//vMfSVccpZdTWFiI1NRUCIIAW1tbSR/Hzp8/X+11UVER0tLSxEXVpLxgKv/3WVd7LPIRWTVoaulXTdFEj6g///wTkyZNQr9+/TB+/HiMGTMG1tbWCAkJqbPtMJVp3rw5rly5UtvVIJROo//GG2/IUnZlCUSlUmH16tVwcnKSNFZ96LHIBFMNmlr6VVM00SMqIyNDHH/QoEEDcbGmd955B/PmzZNsbfHaVFBQgP379+P111+v7apQLdDV1cWwYcOwZMkS9O/fX7Jy60OPRSaYatDU0q+apFKpEBMTozbQ0tXVVbLFs1577TUxqZibmyMlJQXNmjXDo0eP6uRAy6fnbxMEAYWFhdDT08OMGTNqsWZUmx49eiT+nkvpyZMniI6ORmpqKhQKBZo2bYqePXtKumyHnJhgquHx48c4fvy47Eu/asqNGzfw3XffQaVSifOCHT58GL///jtmzZolyazNTk5OuHjxIuzs7NCjRw+sX78esbGxiIuLU1t2uK54+mespaUFY2Nj2Nvbs4vy/4Cnp4QqW3QwJiZGsiXTy6SmpmLx4sXIz88X/z6joqKwY8cOfPPNNxVmx3gVsZG/Gp5u4HtaXesRNWvWLFhZWVWYiHLVqlXIyMhAYGBgjWPk5uZCpVLB3NxcnLIjMTERNjY2GD58uOSjn+WWmpoKLS0tcc2R2NhYHDlyBE2bNsWQIUPEzhJUP02fPl3tddkFRtu2bTFs2DBxIlkpLFy4EHp6evDz84OBgQGA0lVuV65ciaKiIsnGqsmJdzDVUNcSyIukpKTAz8+vwkSUI0aMUJsRuSaUSqX4paulpYWWLVsiOTkZRkZGkv4xasqqVaswcOBANG7cGPfv38cPP/yANm3aICIiAvn5+RgzZkxtV5Fk9Msvv2gsVmJiIgICAsTkAgD/r717DWnyfeMA/nU6tSwP5SR0IampVNgLi6gMFetNWlSIp4w0E1+k0IEO9qYoMMyOUkmGU0iYRgR5gBpGS5MMwkglo4SMnIqzVaJLU+fvRfT8f/trv1bu2cF9P+CLPXO7r4R27b7v57ru+fPnIzU11S6SCwDw65YDCwgIgE6nm3b98+fPZjsVsKSkBO/fvwcA4QN5ZGQEKpUKVVVVZhnDkjQajdASpKWlBSEhIcjPz0dubi6am5utHB3NJVKpFCMjI9Ou6/V67sGQ7UtJSUF5eTkSExONGlHeu3cPaWlpRm1R/nZ/4VcfyB0dHSgpKbG7b/wGgwEuLj/+23R0dAjr7kuWLMGXL1+sGRpZiKWKrSMjI3Hz5k3k5OQIPdbevn2L0tJSsxVBi40JxoEVFhYCAIqLi6c9d/78eaPHf9sCZa59IC9duhQqlQqRkZFob28XEqROpxNOOaS563fF1uaUmZmJ69ev49SpU0aF0GvWrEFGRoZo45oTE4wDs8Se0lz7QN69ezeKiopQW1uL6Oho4e6eFy9eiFbcR7bDksXWHh4eOHbsGPr7+4XblAMCAuyqNxkTjANbsWKF6GPMtQ/kFStWoKysDHq93mjZcPPmzcLRszR3WbrYur6+HnV1dcJe6aJFixAfH4/4+Hi76E3G25QJOp1uxkaU5kpABoNh2gfywMAA3Nzc4OXlZZYxiCxBqVTC2dnZIsXWlZWVaGhowPbt2432YGpraxEXF4f09HTRY5gtJhgHptPpUFxc/MseWmw9T2TcQXlqagpPnz6FXC4Xvdg6MzMTOTk505bjWlpaUFpa+tvOzraAS2QOrKKiAhKJBJcuXUJ+fj5OnjyJr1+/4s6dO9i7d6+1wyOyCR8/fjR6/HOJrLe3V/Sxfy4p//81e5kXMME4sM7OTpw4cQIBAQFwcnKCp6cnwsPDIZVKUV1dbZetXIjMzVoF1tHR0Xj48CEyMzONrqtUKmzatMkqMf0pJhgH9v37dyxcuBDAjzqXoaEh+Pv7Qy6X48OHD1aOjsj23LhxA5mZmdO6UIyOjkKhUMz6TKh/L3sZDAY0NTXh1atXQp1aV1cXdDqd3SQYVvI7MH9/f2GaHxgYCJVKBa1WiwcPHmDRokVWjo7I9jx58uSXx4w3NjbO+v0/fvwo/Gg0GgQFBcHHxweDg4MYHByEt7c3goKCoNFoZj2WJXAG48C2bt0qFDsmJiaioKAAzc3NcHFxmdbUj8iRDQ8PC/seIyMjRpv7BoMBra2tZrkjcq71O+RdZCQYGxuDRqOBr6+vXRZBEoklOTn5P593cnJCUlISdu3aZaGI7AMTjIOzVF8lInv2+vVrTE1N4cyZMzhy5IhRTZeLiwt8fX25rDwDJhgH9ru+SrPdsCSaa7RaLRYvXsxzf0zEPRgHZsm+SkRzgUwmw9jYGLq7u2ec9a9bt85KkdkmJhgHZum+SkT2rq2tDVevXjU6yuLf2P3CGJfIHJgl+yoRzQWHDx9GcHAwUlNTuediAs5gHMxMfZXa29tF76tENBdotVocO3aMycVETDAOxpp9lYjsXVhYGHp7e+3qTBZr4hIZEZGJnj9/jqqqKiQkJMw46w8KCrJSZLaJCYaIyES/K7jkJr8xJhgiIhNptdr/fF4mk1koEvvAaiEiIhPJZDL09PSgrKwMBQUFcHJygkwmQ1tbG/r7+60dns1hgiEiMlFTUxMuX76MJUuWYGBgAJOTkwB+1JTV1NRYOTrbwwRDRGSimpoa5OTkICMjw2iDf/ny5eju7rZeYDaKCYaIyER9fX0IDQ2ddt3d3R16vd4KEdk2JhgiIhP5+Pigr69v2vXOzk7WxsyACYaIyESbN29GeXk53rx5AwD49OkT1Go1KisrsWXLFitHZ3t4mzIR0R9QKpWor6/H+Pg4gB/nwWzbtg0pKSlWjsz2MMEQEf2hsbEx9PT0YGpqCnK5HO7u7tYOySYxwRARkSi4B0NERKJggiEiIlEwwRDZsdLSUty9e9faYRDNiHswRGZw4MABfPnyBRLJ/76zxcTEICsry2xjqNVqPHr0CGfPnjXbexKJiQeOEZnJ8ePHERERYe0wiGwGEwyRiH7OOoKDg6FWq7FgwQLk5eWhr68P1dXVGB8fR3p6OmJiYgAAer0eCoUCL1++hJubG+Li4rBz50709vbi1q1bmJiYwJ49e+Ds7IyKigpcv34dixcvFmowGhoacP/+fQwPDyM8PBzZ2dnC8b5JSUnYv38/6urqMDQ0hKioKGRlZcHJyclafx6a47gHQySyd+/eITAwEAqFAlFRUbhy5Qq6urpQXFyMvLw8KBQKjI6OAgAUCgX0ej2uXbuG06dPo7GxEWq1GnK5HNnZ2QgNDcXt27dRUVExbZyOjg4olUocOnQIpaWlkMlkuHr1qtHvtLa24ty5c7hw4QKePXuGV69eWeJPQA6KCYbITIqKipCRkSH8NDQ0AAD8/PwQGxsLiUSCDRs24NOnT0hMTIRUKsXq1avh4uKC/v5+GAwGNDc3Iy0tDfPmzYOfnx8SEhLQ2Nho0vhNTU2IjY1FUFAQpFIp0tLS8PbtWwwMDAi/s2PHDnh4eMDX1xcrV65kB2ASFZfIiMzk6NGj0/Zg1Go1vLy8hMeurq4AAG9vb6Nro6OjGBoawuTkJHx9fYXnZDIZdDqdSeN//vwZy5YtEx67u7tjwYIF0Ol08PPzmzaum5ubMHMiEgNnMEQ2wtPTE87OzhgcHBSuDQ4OCnsov+Pj42P02tHRUQwPD5v8eiJzY4IhshESiQTr16+HUqnEt2/foNVqUVdXh02bNgH4MfvQ6XSYmJiY8fUbN27E48eP0d3djfHxcSiVSoSEhAizFyJL4xIZkZkUFhYa1cFERERg7dq1f/Qe+/btg0KhQG5uLlxdXREXF4fY2FgAwKpVq4TNfolEgrKyMqPXRkREIDk5GRcvXsTw8DDCwsJw8ODB2f/DiP4SCy2JiEgUXCIjIiJRMMEQEZEomGCIiEgUTDBERCQKJhgiIhIFEwwREYmCCYaIiETBBENERKL4B7qqgJApahB8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.sentiment.value_counts().plot.bar(color='green', fontsize=14)\n",
    "plt.title(\"Emotions distribution\")\n",
    "plt.ylabel(\"Counts\")\n",
    "plt.xlabel(\"Emotion\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YK2la0cESia6",
    "outputId": "10d13c39-f99d-43be-87ac-7dcbe3cce98e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sentiment \n",
       "neutral       8638\n",
       "worry         8459\n",
       "happiness     5209\n",
       "sadness       5165\n",
       "love          3842\n",
       "surprise      2187\n",
       "fun           1776\n",
       "relief        1526\n",
       "hate          1323\n",
       "empty          827\n",
       "enthusiasm     759\n",
       "boredom        179\n",
       "anger          110\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['sentiment']].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VkUqYUncjsaj"
   },
   "source": [
    "<h2> 3. Pre-processing and data selection </h2>\n",
    "    \n",
    "\n",
    "* Here we will first look for emoticons or/and emojis in the content. Then we will replace any emoticons or emojis with text by using the emot library.\n",
    "\n",
    "* The similarity amongst emotions will be computed by using Google-news and Twitter pretrained vector models. \n",
    "* We will group the labels that are closer in similarity except if they have opposite meaning.\n",
    "* Finally, we will drop label or labels that are highly similar with all other emotion labels.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TDWDTw5FZSEw",
    "outputId": "94ee35c8-be61-4cd4-b80b-740ab4625591"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anger: 110 tweets\n",
      "Boredom: 179 tweets\n"
     ]
    }
   ],
   "source": [
    "# Extract the number of tweets of emotions with less instances in the dataset\n",
    "anger = df[df['sentiment']== 'anger']\n",
    "print('Anger: {} tweets'.format(len(anger)))\n",
    "boredom = df[df['sentiment']== 'boredom']\n",
    "print('Boredom: {} tweets'.format(len(boredom)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-oNtZEQv1QYe"
   },
   "source": [
    "<h3> 3.1 Emoticon dictionary </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "lFp0Lpu13_r6"
   },
   "outputs": [],
   "source": [
    "emot_dict = emot.emo_unicode.EMOTICONS_EMO\n",
    "# From Wikipedia list of emoticons correct to angry, \n",
    "# https://en.wikipedia.org/wiki/List_of_emoticons\n",
    "emot_dict = {k: ('angry' if k in ['>-(', '>:(', '>:['] else v) for k, v in emot_dict.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "lNVbRBdtu5YS"
   },
   "outputs": [],
   "source": [
    "# From wikipedia remove andry \n",
    "emot_dict = {k: ('Frown, sad or pouting' if v == 'Frown, sad, andry or pouting' \n",
    "                 else v) for k, v in emot_dict.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zYnzz3Hzu_yj"
   },
   "source": [
    "<h3> 3.2 Find emoticons in content </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "OBHYMy8Ju_CV"
   },
   "outputs": [],
   "source": [
    "# Find the emoticons used in the tweets and their categorical label\n",
    "emo = defaultdict(list)\n",
    "for k, v in emot_dict.items():\n",
    "    for i, l in enumerate(list(df.content)):\n",
    "        a = df.sentiment.iloc[i]\n",
    "        if k in l and a not in emo[k]:\n",
    "            emo[k].append(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XTiQVDURvU5h",
    "outputId": "f3ef7391-1f93-4697-b000-18fbbfdb36c2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A total 53 emoticons were used in the dataset.\n"
     ]
    }
   ],
   "source": [
    "print('A total {} emoticons were used in the dataset.'.format(len(emo)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 300
    },
    "id": "y1q4Tc2tvYfd",
    "outputId": "2366def9-f91d-4242-d3f0-32ae380d4116"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-43776094-6ccd-4caf-aaf6-987aed564d67\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:]</th>\n",
       "      <th>:3</th>\n",
       "      <th>8-)</th>\n",
       "      <th>:o)</th>\n",
       "      <th>:^)</th>\n",
       "      <th>=]</th>\n",
       "      <th>8D</th>\n",
       "      <th>XD</th>\n",
       "      <th>=D</th>\n",
       "      <th>=3</th>\n",
       "      <th>...</th>\n",
       "      <th>:&amp;</th>\n",
       "      <th>0:3</th>\n",
       "      <th>%)</th>\n",
       "      <th>;_;</th>\n",
       "      <th>;;</th>\n",
       "      <th>T.T</th>\n",
       "      <th>QQ</th>\n",
       "      <th>o_0</th>\n",
       "      <th>o.O</th>\n",
       "      <th>oO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>worry</td>\n",
       "      <td>neutral</td>\n",
       "      <td>sadness</td>\n",
       "      <td>sadness</td>\n",
       "      <td>surprise</td>\n",
       "      <td>neutral</td>\n",
       "      <td>sadness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>...</td>\n",
       "      <td>worry</td>\n",
       "      <td>worry</td>\n",
       "      <td>neutral</td>\n",
       "      <td>sadness</td>\n",
       "      <td>surprise</td>\n",
       "      <td>sadness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>worry</td>\n",
       "      <td>fun</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>worry</td>\n",
       "      <td>worry</td>\n",
       "      <td>happiness</td>\n",
       "      <td>love</td>\n",
       "      <td>neutral</td>\n",
       "      <td>happiness</td>\n",
       "      <td>worry</td>\n",
       "      <td>worry</td>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>surprise</td>\n",
       "      <td>surprise</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>sadness</td>\n",
       "      <td>relief</td>\n",
       "      <td>NaN</td>\n",
       "      <td>worry</td>\n",
       "      <td>NaN</td>\n",
       "      <td>surprise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sadness</td>\n",
       "      <td>sadness</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fun</td>\n",
       "      <td>neutral</td>\n",
       "      <td>love</td>\n",
       "      <td>happiness</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>love</td>\n",
       "      <td>sadness</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>worry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>love</td>\n",
       "      <td>empty</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>worry</td>\n",
       "      <td>relief</td>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>neutral</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>happiness</td>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>worry</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>neutral</td>\n",
       "      <td>happiness</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>love</td>\n",
       "      <td>happiness</td>\n",
       "      <td>hate</td>\n",
       "      <td>love</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>hate</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 53 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-43776094-6ccd-4caf-aaf6-987aed564d67')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-43776094-6ccd-4caf-aaf6-987aed564d67 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-43776094-6ccd-4caf-aaf6-987aed564d67');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "          :]         :3        8-)      :o)      :^)         =]         8D  \\\n",
       "0  happiness    neutral      worry  neutral  sadness    sadness   surprise   \n",
       "1      worry      worry  happiness     love  neutral  happiness      worry   \n",
       "2    sadness    sadness        NaN      NaN      NaN        fun    neutral   \n",
       "3       love      empty        NaN      NaN      NaN      worry     relief   \n",
       "4    neutral  happiness        NaN      NaN      NaN       love  happiness   \n",
       "\n",
       "           XD          =D       =3  ...         :&         0:3       %)  \\\n",
       "0     neutral     sadness  neutral  ...      worry       worry  neutral   \n",
       "1       worry  enthusiasm      NaN  ...   surprise    surprise      NaN   \n",
       "2        love   happiness      NaN  ...       love     sadness      NaN   \n",
       "3  enthusiasm     neutral      NaN  ...  happiness  enthusiasm      NaN   \n",
       "4        hate        love      NaN  ...    sadness     neutral      NaN   \n",
       "\n",
       "       ;_;         ;;      T.T       QQ      o_0    o.O        oO  \n",
       "0  sadness   surprise  sadness  neutral  neutral  worry       fun  \n",
       "1      NaN    sadness   relief      NaN    worry    NaN  surprise  \n",
       "2      NaN  happiness  neutral      NaN      NaN    NaN     worry  \n",
       "3      NaN      worry      NaN      NaN      NaN    NaN   neutral  \n",
       "4      NaN       hate      NaN      NaN      NaN    NaN   sadness  \n",
       "\n",
       "[5 rows x 53 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show the emoticon and the content class label\n",
    "data = pd.DataFrame(dict([(k,pd.Series(v)) for k,v in emo.items()]))\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "co0DM7qgvYrt",
    "outputId": "78282fee-6cb9-452d-cb06-d95df9b03c1f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 13 entries, 0 to 12\n",
      "Data columns (total 53 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   :]      10 non-null     object\n",
      " 1   :3      12 non-null     object\n",
      " 2   8-)     2 non-null      object\n",
      " 3   :o)     2 non-null      object\n",
      " 4   :^)     2 non-null      object\n",
      " 5   =]      6 non-null      object\n",
      " 6   8D      5 non-null      object\n",
      " 7   XD      10 non-null     object\n",
      " 8   =D      9 non-null      object\n",
      " 9   =3      1 non-null      object\n",
      " 10  :c      1 non-null      object\n",
      " 11  :[      4 non-null      object\n",
      " 12  :{      1 non-null      object\n",
      " 13  :@      6 non-null      object\n",
      " 14  :'(     6 non-null      object\n",
      " 15  :')     3 non-null      object\n",
      " 16  D:      9 non-null      object\n",
      " 17  D8      1 non-null      object\n",
      " 18  D;      3 non-null      object\n",
      " 19  D=      2 non-null      object\n",
      " 20  DX      3 non-null      object\n",
      " 21  :O      8 non-null      object\n",
      " 22  :o      7 non-null      object\n",
      " 23  :-0     1 non-null      object\n",
      " 24  :-*     1 non-null      object\n",
      " 25  :*      5 non-null      object\n",
      " 26  :X      3 non-null      object\n",
      " 27  ;)      12 non-null     object\n",
      " 28  *)      1 non-null      object\n",
      " 29  ;]      3 non-null      object\n",
      " 30  ;D      8 non-null      object\n",
      " 31  XP      8 non-null      object\n",
      " 32  :b      2 non-null      object\n",
      " 33  d:      7 non-null      object\n",
      " 34  =p      4 non-null      object\n",
      " 35  :/      13 non-null     object\n",
      " 36  =/      7 non-null      object\n",
      " 37  :L      7 non-null      object\n",
      " 38  :S      11 non-null     object\n",
      " 39  :|      9 non-null      object\n",
      " 40  :$      1 non-null      object\n",
      " 41  :x      1 non-null      object\n",
      " 42  :#      1 non-null      object\n",
      " 43  :&      6 non-null      object\n",
      " 44  0:3     7 non-null      object\n",
      " 45  %)      1 non-null      object\n",
      " 46  ;_;     1 non-null      object\n",
      " 47  ;;      7 non-null      object\n",
      " 48  T.T     3 non-null      object\n",
      " 49  QQ      1 non-null      object\n",
      " 50  o_0     2 non-null      object\n",
      " 51  o.O     1 non-null      object\n",
      " 52  oO      7 non-null      object\n",
      "dtypes: object(53)\n",
      "memory usage: 5.5+ KB\n"
     ]
    }
   ],
   "source": [
    "# Show information of the emoticons and number of different content labels that they appear\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RIyVGrt-vb8Q",
    "outputId": "16694e60-29f1-45d2-e4b7-1b66435bc2cb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emoticon :] - Happy face or smiley appears in content with 10 different labels.\n",
      "Labels: ['happiness', 'worry', 'sadness', 'love', 'neutral', 'empty', 'enthusiasm', 'surprise', 'boredom', 'fun']\n",
      "Emoticon :3 - Happy face smiley appears in content with 12 different labels.\n",
      "Labels: ['neutral', 'worry', 'sadness', 'empty', 'happiness', 'love', 'surprise', 'hate', 'boredom', 'relief', 'enthusiasm', 'fun']\n",
      "Emoticon =] - Happy face smiley appears in content with 6 different labels.\n",
      "Labels: ['sadness', 'happiness', 'fun', 'worry', 'love', 'neutral']\n",
      "Emoticon 8D - Laughing, big grin or laugh with glasses appears in content with 5 different labels.\n",
      "Labels: ['surprise', 'worry', 'neutral', 'relief', 'happiness']\n",
      "Emoticon XD - Laughing, big grin or laugh with glasses appears in content with 10 different labels.\n",
      "Labels: ['neutral', 'worry', 'love', 'enthusiasm', 'hate', 'happiness', 'surprise', 'relief', 'empty', 'fun']\n",
      "Emoticon =D - Laughing, big grin or laugh with glasses appears in content with 9 different labels.\n",
      "Labels: ['sadness', 'enthusiasm', 'happiness', 'neutral', 'love', 'surprise', 'fun', 'worry', 'relief']\n",
      "Emoticon :@ - Frown, sad or pouting appears in content with 6 different labels.\n",
      "Labels: ['neutral', 'worry', 'love', 'sadness', 'hate', 'empty']\n",
      "Emoticon :'( - Crying appears in content with 6 different labels.\n",
      "Labels: ['sadness', 'worry', 'hate', 'surprise', 'enthusiasm', 'neutral']\n",
      "Emoticon D: - Sadness appears in content with 9 different labels.\n",
      "Labels: ['neutral', 'worry', 'fun', 'love', 'boredom', 'sadness', 'happiness', 'hate', 'surprise']\n",
      "Emoticon :O - Surprise appears in content with 8 different labels.\n",
      "Labels: ['sadness', 'love', 'worry', 'surprise', 'neutral', 'happiness', 'hate', 'fun']\n",
      "Emoticon :o - Surprise appears in content with 7 different labels.\n",
      "Labels: ['neutral', 'happiness', 'sadness', 'worry', 'surprise', 'love', 'enthusiasm']\n",
      "Emoticon :* - Kiss appears in content with 5 different labels.\n",
      "Labels: ['sadness', 'happiness', 'love', 'neutral', 'surprise']\n",
      "Emoticon ;) - Wink or smirk appears in content with 12 different labels.\n",
      "Labels: ['fun', 'worry', 'relief', 'love', 'surprise', 'happiness', 'sadness', 'neutral', 'boredom', 'enthusiasm', 'hate', 'empty']\n",
      "Emoticon ;D - Wink or smirk appears in content with 8 different labels.\n",
      "Labels: ['sadness', 'worry', 'neutral', 'happiness', 'surprise', 'fun', 'relief', 'love']\n",
      "Emoticon XP - Tongue sticking out, cheeky, playful or blowing a raspberry appears in content with 8 different labels.\n",
      "Labels: ['worry', 'fun', 'anger', 'sadness', 'neutral', 'empty', 'relief', 'happiness']\n",
      "Emoticon d: - Tongue sticking out, cheeky, playful or blowing a raspberry appears in content with 7 different labels.\n",
      "Labels: ['sadness', 'worry', 'relief', 'neutral', 'empty', 'love', 'happiness']\n",
      "Emoticon :/ - Skeptical, annoyed, undecided, uneasy or hesitant appears in content with 13 different labels.\n",
      "Labels: ['worry', 'neutral', 'sadness', 'hate', 'love', 'empty', 'surprise', 'relief', 'happiness', 'enthusiasm', 'fun', 'anger', 'boredom']\n",
      "Emoticon =/ - Skeptical, annoyed, undecided, uneasy or hesitant appears in content with 7 different labels.\n",
      "Labels: ['worry', 'hate', 'surprise', 'sadness', 'neutral', 'fun', 'happiness']\n",
      "Emoticon :L - Skeptical, annoyed, undecided, uneasy or hesitant appears in content with 7 different labels.\n",
      "Labels: ['worry', 'sadness', 'love', 'surprise', 'neutral', 'happiness', 'empty']\n",
      "Emoticon :S - Skeptical, annoyed, undecided, uneasy or hesitant appears in content with 11 different labels.\n",
      "Labels: ['sadness', 'happiness', 'love', 'worry', 'hate', 'neutral', 'relief', 'surprise', 'enthusiasm', 'fun', 'boredom']\n",
      "Emoticon :| - Straight face appears in content with 9 different labels.\n",
      "Labels: ['worry', 'love', 'sadness', 'happiness', 'neutral', 'surprise', 'enthusiasm', 'fun', 'relief']\n",
      "Emoticon :& - Sealed lips or wearing braces or tongue-tied appears in content with 6 different labels.\n",
      "Labels: ['worry', 'surprise', 'love', 'happiness', 'sadness', 'empty']\n",
      "Emoticon 0:3 - Angel, saint or innocent appears in content with 7 different labels.\n",
      "Labels: ['worry', 'surprise', 'sadness', 'enthusiasm', 'neutral', 'fun', 'relief']\n",
      "Emoticon ;; - Sad or Crying appears in content with 7 different labels.\n",
      "Labels: ['surprise', 'sadness', 'happiness', 'worry', 'hate', 'neutral', 'love']\n",
      "Emoticon oO - Surprised appears in content with 7 different labels.\n",
      "Labels: ['fun', 'surprise', 'worry', 'neutral', 'sadness', 'empty', 'happiness']\n"
     ]
    }
   ],
   "source": [
    "# Find emoticons with content labelled with 5 or more categories\n",
    "for k,v in emo.items():\n",
    "    if len(v) >= 5:\n",
    "        print('Emoticon {0} - {1} appears in content with {2} different labels.'.format(k, emot_dict[k], len(v)))\n",
    "        print('Labels: {}'.format(v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NAk13HgAvydF"
   },
   "source": [
    "<h3> 3.3 Find tweets with emojis </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PX-I1oYdvxlQ",
    "outputId": "94398696-1fbb-4c9a-856a-cbc4e6adfc98"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No emojies in the dataset\n"
     ]
    }
   ],
   "source": [
    "emoji = list()\n",
    "for i, l in enumerate(list(df.content)):\n",
    "    if 'U+'in l:\n",
    "        emoji.append(i)\n",
    "if len(emoji)==0:\n",
    "    print('No emojies in the dataset')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "g9lzM_j21QYg"
   },
   "source": [
    "The dataset content did not have emojis."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OJEQjt_c1QYg"
   },
   "source": [
    "<h3> 3.4 Finding similarity in emotions using Gensim </h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BnjfojU3wPld"
   },
   "source": [
    "From the section 3.2 we observed that some emoticons appear in content that has been labelled differently.  For instance the emoticon happy face or smiley appears in content labelled as 'happiness', 'worry', 'sadness', 'love', 'neutral', 'empty', 'enthusiasm', 'surprise', 'boredom' and 'fun'. These\n",
    "findings show that the emoticon and content can be ambiguous, expressing different feelings at\n",
    "once. It also could be mislabelled or the incorrect emoticon used since the emoticons were manually input rather than automatically generated from text. Therefore, this dataset represents a challenge for a classification task. To address this issue we will follow the strategy below:\n",
    "\n",
    "* Replace the following labels that are adjectives and/or verbs with nouns; empty with emptiness, and hate with hatred.\n",
    "* Load pretrained word2vec-google-news-300 and 'glove-twitter-100 vectors and compute similitariy amongst the categorical labels (emotion).\n",
    "* Join similar emotions \n",
    "* Remove instances of labels that are related with all labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "id": "5-klDDmxwd8E"
   },
   "outputs": [],
   "source": [
    "df_emotions = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "id": "nyYQhMdvwocC"
   },
   "outputs": [],
   "source": [
    "df_emotions.loc[(df_emotions.sentiment == 'empty'), 'sentiment'] = 'emptiness'\n",
    "df_emotions.loc[(df_emotions.sentiment == 'hate'), 'sentiment'] = 'hatred'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "id": "UpVBNf4KwtrL"
   },
   "outputs": [],
   "source": [
    "emotions_m = sorted(list(df_emotions.sentiment.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "id": "1LxNdNHQwRWX"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "# Load google-new word2vec vectors\n",
    "gwv = api.load('word2vec-google-news-300')\n",
    "\n",
    "# Load glove-twitter vectors \n",
    "glove_vectors = api.load('glove-twitter-100') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "id": "zn2CVI_5wyaK"
   },
   "outputs": [],
   "source": [
    "# Compute similarity from pre-trained models\n",
    "sim_g ={}\n",
    "sim_t ={}\n",
    "\n",
    "for e in emotions_m:\n",
    "    for i in range(len(emotions_m)-1):\n",
    "        # google news\n",
    "        w1 = e\n",
    "        w2 = emotions_m[i+1] \n",
    "        if w1 == w2:\n",
    "            continue    \n",
    "        key = (w1, w2)\n",
    "        if key[::-1] in sim_g.keys(): \n",
    "              continue\n",
    "        else: \n",
    "            # google vectors\n",
    "            similarity_g = gwv.similarity(w1, w2)\n",
    "            # twitter vectors\n",
    "            similarity_t = glove_vectors.similarity(w1, w2)\n",
    "            sim_g[key] = similarity_g\n",
    "            sim_t[key] = similarity_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 677
    },
    "id": "qIT3nL3Owyel",
    "outputId": "053a7ff4-4ede-4392-d0d2-1aecf75b2d40"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-29ad11e9-0700-40bd-8da3-647ff80e1b74\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>twitter</th>\n",
       "      <th>goog_news</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(anger, hatred)</th>\n",
       "      <td>0.768769</td>\n",
       "      <td>0.590178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(emptiness, sadness)</th>\n",
       "      <td>0.550510</td>\n",
       "      <td>0.589492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(fun, love)</th>\n",
       "      <td>0.710840</td>\n",
       "      <td>0.466562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(anger, sadness)</th>\n",
       "      <td>0.702778</td>\n",
       "      <td>0.451019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(happiness, love)</th>\n",
       "      <td>0.628533</td>\n",
       "      <td>0.450275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(boredom, emptiness)</th>\n",
       "      <td>0.361649</td>\n",
       "      <td>0.450142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(emptiness, happiness)</th>\n",
       "      <td>0.430916</td>\n",
       "      <td>0.439732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(happiness, sadness)</th>\n",
       "      <td>0.724261</td>\n",
       "      <td>0.426663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(anger, enthusiasm)</th>\n",
       "      <td>0.503019</td>\n",
       "      <td>0.408466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(hatred, sadness)</th>\n",
       "      <td>0.627424</td>\n",
       "      <td>0.395898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(love, sadness)</th>\n",
       "      <td>0.457202</td>\n",
       "      <td>0.367016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(enthusiasm, love)</th>\n",
       "      <td>0.194183</td>\n",
       "      <td>0.360807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(emptiness, hatred)</th>\n",
       "      <td>0.499148</td>\n",
       "      <td>0.359813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(hatred, love)</th>\n",
       "      <td>0.357668</td>\n",
       "      <td>0.348797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(boredom, happiness)</th>\n",
       "      <td>0.484366</td>\n",
       "      <td>0.347592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(enthusiasm, hatred)</th>\n",
       "      <td>0.519395</td>\n",
       "      <td>0.345579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(boredom, sadness)</th>\n",
       "      <td>0.522585</td>\n",
       "      <td>0.342371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(boredom, fun)</th>\n",
       "      <td>0.363095</td>\n",
       "      <td>0.340451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(enthusiasm, sadness)</th>\n",
       "      <td>0.407770</td>\n",
       "      <td>0.328432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(love, worry)</th>\n",
       "      <td>0.646493</td>\n",
       "      <td>0.314854</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-29ad11e9-0700-40bd-8da3-647ff80e1b74')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-29ad11e9-0700-40bd-8da3-647ff80e1b74 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-29ad11e9-0700-40bd-8da3-647ff80e1b74');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                         twitter  goog_news\n",
       "(anger, hatred)         0.768769   0.590178\n",
       "(emptiness, sadness)    0.550510   0.589492\n",
       "(fun, love)             0.710840   0.466562\n",
       "(anger, sadness)        0.702778   0.451019\n",
       "(happiness, love)       0.628533   0.450275\n",
       "(boredom, emptiness)    0.361649   0.450142\n",
       "(emptiness, happiness)  0.430916   0.439732\n",
       "(happiness, sadness)    0.724261   0.426663\n",
       "(anger, enthusiasm)     0.503019   0.408466\n",
       "(hatred, sadness)       0.627424   0.395898\n",
       "(love, sadness)         0.457202   0.367016\n",
       "(enthusiasm, love)      0.194183   0.360807\n",
       "(emptiness, hatred)     0.499148   0.359813\n",
       "(hatred, love)          0.357668   0.348797\n",
       "(boredom, happiness)    0.484366   0.347592\n",
       "(enthusiasm, hatred)    0.519395   0.345579\n",
       "(boredom, sadness)      0.522585   0.342371\n",
       "(boredom, fun)          0.363095   0.340451\n",
       "(enthusiasm, sadness)   0.407770   0.328432\n",
       "(love, worry)           0.646493   0.314854"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sort by google-news similarity\n",
    "similarity= pd.DataFrame([sim_t, sim_g]).T\n",
    "similarity.columns = ['twitter', 'goog_news']\n",
    "similarity = similarity.sort_values(by= 'goog_news', ascending = False)\n",
    "similarity.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 677
    },
    "id": "nZa-IWo_wyh2",
    "outputId": "23d1d515-29e6-4556-9868-fca2a968c19c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-6ee4ccbf-0ae1-4063-a2c5-3497bb63b868\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>twitter</th>\n",
       "      <th>goog_news</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(anger, hatred)</th>\n",
       "      <td>0.768769</td>\n",
       "      <td>0.590178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(happiness, sadness)</th>\n",
       "      <td>0.724261</td>\n",
       "      <td>0.426663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(fun, love)</th>\n",
       "      <td>0.710840</td>\n",
       "      <td>0.466562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(anger, sadness)</th>\n",
       "      <td>0.702778</td>\n",
       "      <td>0.451019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(love, worry)</th>\n",
       "      <td>0.646493</td>\n",
       "      <td>0.314854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(fun, worry)</th>\n",
       "      <td>0.637802</td>\n",
       "      <td>0.272918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(happiness, worry)</th>\n",
       "      <td>0.630607</td>\n",
       "      <td>0.191000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(happiness, love)</th>\n",
       "      <td>0.628533</td>\n",
       "      <td>0.450275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(hatred, sadness)</th>\n",
       "      <td>0.627424</td>\n",
       "      <td>0.395898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(fun, surprise)</th>\n",
       "      <td>0.619150</td>\n",
       "      <td>0.202897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(anger, happiness)</th>\n",
       "      <td>0.593026</td>\n",
       "      <td>0.258086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(anger, boredom)</th>\n",
       "      <td>0.576564</td>\n",
       "      <td>0.276181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(happiness, hatred)</th>\n",
       "      <td>0.554690</td>\n",
       "      <td>0.290261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(emptiness, sadness)</th>\n",
       "      <td>0.550510</td>\n",
       "      <td>0.589492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(love, surprise)</th>\n",
       "      <td>0.541222</td>\n",
       "      <td>0.154405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(boredom, sadness)</th>\n",
       "      <td>0.522585</td>\n",
       "      <td>0.342371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(enthusiasm, hatred)</th>\n",
       "      <td>0.519395</td>\n",
       "      <td>0.345579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(surprise, worry)</th>\n",
       "      <td>0.505873</td>\n",
       "      <td>0.275078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(anger, enthusiasm)</th>\n",
       "      <td>0.503019</td>\n",
       "      <td>0.408466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(fun, happiness)</th>\n",
       "      <td>0.501005</td>\n",
       "      <td>0.279264</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6ee4ccbf-0ae1-4063-a2c5-3497bb63b868')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-6ee4ccbf-0ae1-4063-a2c5-3497bb63b868 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-6ee4ccbf-0ae1-4063-a2c5-3497bb63b868');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                       twitter  goog_news\n",
       "(anger, hatred)       0.768769   0.590178\n",
       "(happiness, sadness)  0.724261   0.426663\n",
       "(fun, love)           0.710840   0.466562\n",
       "(anger, sadness)      0.702778   0.451019\n",
       "(love, worry)         0.646493   0.314854\n",
       "(fun, worry)          0.637802   0.272918\n",
       "(happiness, worry)    0.630607   0.191000\n",
       "(happiness, love)     0.628533   0.450275\n",
       "(hatred, sadness)     0.627424   0.395898\n",
       "(fun, surprise)       0.619150   0.202897\n",
       "(anger, happiness)    0.593026   0.258086\n",
       "(anger, boredom)      0.576564   0.276181\n",
       "(happiness, hatred)   0.554690   0.290261\n",
       "(emptiness, sadness)  0.550510   0.589492\n",
       "(love, surprise)      0.541222   0.154405\n",
       "(boredom, sadness)    0.522585   0.342371\n",
       "(enthusiasm, hatred)  0.519395   0.345579\n",
       "(surprise, worry)     0.505873   0.275078\n",
       "(anger, enthusiasm)   0.503019   0.408466\n",
       "(fun, happiness)      0.501005   0.279264"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sort by twitter similarity\n",
    "similarity_t = similarity.copy()\n",
    "similarity_t =similarity_t.sort_values(by= 'twitter', ascending = False)\n",
    "similarity_t.head(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SS4BP3YIwylT",
    "outputId": "ea56cedc-f1f3-4d17-c201-d1613f5d025d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('anger', 'relief') 0.4591962\n",
      "('boredom', 'relief') 0.3327248\n",
      "('emptiness', 'relief') 0.24379823\n",
      "('enthusiasm', 'relief') 0.29858676\n",
      "('fun', 'relief') 0.353736\n",
      "('happiness', 'relief') 0.40495405\n",
      "('hatred', 'relief') 0.40448517\n",
      "('love', 'relief') 0.3014868\n",
      "('neutral', 'relief') 0.27753094\n",
      "('relief', 'sadness') 0.32705474\n",
      "('relief', 'surprise') 0.33689827\n",
      "('relief', 'worry') 0.30643463\n"
     ]
    }
   ],
   "source": [
    "# Similarities to relief\n",
    "for k in sim_t:\n",
    "    if 'relief' in k:\n",
    "        print(k, sim_t[k])\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gyXix6Tfwxge"
   },
   "source": [
    "<h3> 3.5 Aggregating emotions </h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Oh5Qy2zBwxpT"
   },
   "source": [
    "-  **anger** - hatred and anger\n",
    "-  **happiness** - enthusiasm, fun, love, hapiness, relief, surprise\n",
    "-  **neutral** - neutral\n",
    "-  **sadness** - boredom, emptiness and sadness\n",
    "-  **worry** - worry dropped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "id": "HUf9NMDgyFzJ"
   },
   "outputs": [],
   "source": [
    "# Change column name from sentiment to emotion\n",
    "df_emotions.rename(columns = {'sentiment':'emotion'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "id": "B8BAE901aLqL"
   },
   "outputs": [],
   "source": [
    "df_emotions.emotion.replace({'hatred': 'anger', 'emptiness': 'sadness', 'boredom':'sadness', \n",
    "                      'fun': 'happiness', 'love': 'happiness', 'enthusiasm': 'happiness', \n",
    "                      'surprise': 'happiness', 'relief': 'happiness'}, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "id": "X5ekRKSg1A1t"
   },
   "outputs": [],
   "source": [
    "# Removing worry \n",
    "df_emotions = df_emotions[df_emotions.emotion != 'worry']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eezKEKHho-QI"
   },
   "source": [
    "<h3> 3.6 Removing unnecesary column </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "psaO1a5_S9Gb",
    "outputId": "2c10e04a-8664-47a1-f15a-022171a72b29"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-947cb01a-07de-482e-9ba4-5ef78a0905fd\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>emotion</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sadness</td>\n",
       "      <td>@tiffanylue i know  i was listenin to bad habi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sadness</td>\n",
       "      <td>Layin n bed with a headache  ughhhh...waitin o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sadness</td>\n",
       "      <td>Funeral ceremony...gloomy friday...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happiness</td>\n",
       "      <td>wants to hang out with friends SOON!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>neutral</td>\n",
       "      <td>@dannycastillo We want to trade with someone w...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-947cb01a-07de-482e-9ba4-5ef78a0905fd')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-947cb01a-07de-482e-9ba4-5ef78a0905fd button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-947cb01a-07de-482e-9ba4-5ef78a0905fd');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "     emotion                                            content\n",
       "0    sadness  @tiffanylue i know  i was listenin to bad habi...\n",
       "1    sadness  Layin n bed with a headache  ughhhh...waitin o...\n",
       "2    sadness                Funeral ceremony...gloomy friday...\n",
       "3  happiness               wants to hang out with friends SOON!\n",
       "4    neutral  @dannycastillo We want to trade with someone w..."
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_emotions = df_emotions[['emotion','content']]\n",
    "df_emotions.head(5)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gY2yGePqeGK0"
   },
   "source": [
    "Let's review the unique emotions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jJr7QXMFA3Q2",
    "outputId": "80f5b896-b6f9-41d4-c47f-17793c6ab429"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "happiness    15299\n",
       "neutral       8638\n",
       "sadness       6171\n",
       "anger         1433\n",
       "Name: emotion, dtype: int64"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_emotions['emotion'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DkhMgMLkF986",
    "outputId": "87e89ef5-3e59-47d3-b3ac-d7b9bd1a21e9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['anger', 'happiness', 'neutral', 'sadness']\n"
     ]
    }
   ],
   "source": [
    "# Extract the unique emotions\n",
    "emotions = sorted(list(df_emotions['emotion'].unique()))\n",
    "print(emotions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C5WbjyH6hFn-"
   },
   "source": [
    "<h3> 3.7 Assigning  encoded labels </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "70BFEIU3u6mY",
    "outputId": "6aaf847a-6e25-4756-d2e9-07b65871a9cd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'anger': 0, 'happiness': 1, 'neutral': 2, 'sadness': 3}\n"
     ]
    }
   ],
   "source": [
    "# Creating the label dictionary\n",
    "emo_to_label = {em:i for i, em in enumerate(emotions)}\n",
    "print(emo_to_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "id": "pThIADsMmL9v"
   },
   "outputs": [],
   "source": [
    "# Get the class from encoded label\n",
    "label_to_emotion = {i:em for i, em in enumerate(emotions)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UQTVyAc8aREm"
   },
   "source": [
    "<h3> 3.8 Creating the encoded labels</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "id": "zPF-0bvH62y6"
   },
   "outputs": [],
   "source": [
    "label = [emo_to_label[em] for em in df_emotions.emotion]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "id": "pBp3Xd2_mYCb"
   },
   "outputs": [],
   "source": [
    "# df_emotions['label']=label\n",
    "df_emotions.insert(2, 'label', label )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "B3k6hGgC0zb8",
    "outputId": "aa2a0806-f407-42e8-e0b0-3a18f704c2b3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-17b51de3-b9e5-4038-b64f-9f61f0192023\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>emotion</th>\n",
       "      <th>content</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sadness</td>\n",
       "      <td>@tiffanylue i know  i was listenin to bad habi...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sadness</td>\n",
       "      <td>Layin n bed with a headache  ughhhh...waitin o...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sadness</td>\n",
       "      <td>Funeral ceremony...gloomy friday...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happiness</td>\n",
       "      <td>wants to hang out with friends SOON!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>neutral</td>\n",
       "      <td>@dannycastillo We want to trade with someone w...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-17b51de3-b9e5-4038-b64f-9f61f0192023')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-17b51de3-b9e5-4038-b64f-9f61f0192023 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-17b51de3-b9e5-4038-b64f-9f61f0192023');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "     emotion                                            content  label\n",
       "0    sadness  @tiffanylue i know  i was listenin to bad habi...      3\n",
       "1    sadness  Layin n bed with a headache  ughhhh...waitin o...      3\n",
       "2    sadness                Funeral ceremony...gloomy friday...      3\n",
       "3  happiness               wants to hang out with friends SOON!      1\n",
       "4    neutral  @dannycastillo We want to trade with someone w...      2"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_emotions.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UyaoLNJmR45r"
   },
   "source": [
    "<h2> 4. Dataset subsets </h2>\n",
    "\n",
    "Here we will replace any emoticons with test and split the dataset in three subsets: training, validation and test.  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gmGTGjoE10wz"
   },
   "source": [
    "<h3> 4.1 Replace emoticons </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "id": "GZA1WuZ97wP9"
   },
   "outputs": [],
   "source": [
    "def replace_emoticons(text):\n",
    "    \"\"\"\n",
    "    :param text: text to process\n",
    "    :return: text with emoticons converted to text \n",
    "    \"\"\"\n",
    "    for em in emot_dict:\n",
    "        text = text.replace(em, emot_dict[em].replace(\" \",\"_\"))\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "id": "CeCQKOxC976H"
   },
   "outputs": [],
   "source": [
    "# replace emoticons \n",
    "replace_emot = True\n",
    "if replace_emot:\n",
    "    for r, t in enumerate(df_emotions.content):\n",
    "        t = replace_emoticons(t)\n",
    "        df_emotions.iat[r, 1] = t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6F2Fm9Oq-Z0c"
   },
   "source": [
    "<h3> 4.2 Splitting the dataset </h3>\n",
    "\n",
    "To address the class imbalace the dataset will be split by label stratification using scikit-learn train_test_split function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "id": "hV-tojZITejO"
   },
   "outputs": [],
   "source": [
    "X = df_emotions.content.values\n",
    "y = df_emotions.label.values\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.20, stratify=y, \n",
    "                                                  random_state= seed_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "id": "eWg1P0J4Uei_"
   },
   "outputs": [],
   "source": [
    "X_val, X_test, y_val, y_test = train_test_split(X_val, y_val, test_size=0.5, stratify=y_val,\n",
    "                                                random_state= seed_)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rRxPT0Dq5KEP"
   },
   "source": [
    "<h3> 4.3 Visualisation of emotion distribution </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 379
    },
    "id": "uQ0IVmz7rQEy",
    "outputId": "bc0a29c2-4322-4b37-dfea-e5d1695bdd2f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Emotion')"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAFYCAYAAAD6NUMTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVxUVf8H8M8ADvssNiCyjIo4SCkQmvEkhimkuaS59qhpgGEpSU/bLzM1twdsdcsMc8UyMxeUTG0oNSwxNR0BYYglBFSWGBbZhDm/P4j7eAUUdeAy8H2/Xr5ezr3n3vmeA8x3zj3n3CtijDEQQgghAjAROgBCCCGdFyUhQgghgqEkRAghRDCUhAghhAiGkhAhhBDBUBIihBAiGEpCncC2bdtgZmYmdBjNEolE2LlzZ7OvDWno0KGYPXt2s68NrWfPnlixYkWrnf9erVu3Ds7OzjAxMcH777/fZu+7YMECdOvWDSKRCNu2bWuz9zUWL774IgICAlr9fbKysiASiRAfH9/q79VijBjUrFmzGIBG/6ytrVv9va9cucIAsJ9//pm3vaKigl27dq3V3/9+AWDR0dHc66tXr7LKysoWHRsdHc3u5de4qKiIlZSUcK/9/f1ZSEhIy4NtRkhICPP392+0PT8/n5WXlz/w+Q0hNzeXmZiYsPXr17O8vDxWVlbGhg8fzmbNmtWq73v69GkGgB04cIBdvXqVVVRUGOzcpqambOvWrQY7n1BmzZrFhg8fbtBz9u7dmy1ZsoS3rba2ll29epXV1NQY9L0eRPv9emzEhgwZgm+//Za3zcREuE6npaUlLC0tBXv/e+Xg4GDwc9bU1EAsFqNr164GP/ed2NnZten73UlGRgb0ej2effZZdO/e3eDnv3nzJrp06dJoe1paGkxMTDBu3DiDv6chNRd/R2Jqatoqf18PROgs2NG05BuNv78/Cw4OZgsXLmR2dnZMKpWyd999l9XV1bGlS5cye3t7plAo2Lvvvss7rrS0lIWGhjKFQsHEYjEbMGAAO3r0KLcft/W+evTowRhjbOvWrczU1JR3ru+//575+PgwsVjM7Ozs2CuvvML7xt5Qjy+++IIplUpma2vLxo4dy+tRXblyhU2YMIE99NBDzNzcnPXq1Yt98MEHd6z7Tz/9xPr378/Mzc1Z//792U8//dSoJ3T7602bNrG+ffsyc3NzJpfL2ZAhQ9iVK1fYzz//3KjODd/qG9r4vffeYw4ODqxbt27c9lt7Pv7+/iwoKIj93//9H3vooYeYra0te+mll3g9saZ6S8uXL+fad8mSJY3iaPh23qNHD7Z8+fIW/wwzMzMZALZ79242evRoZmlpyXr16nXXb/t///03mz59OnNxcWEWFhZMpVKxjz76iOn1+mZjbKrX3tCLvnbtGps1axZTKBTMxsaGPfHEE+zEiRPc+zW0fWxsLBs8eDAzNzdnGzZsaBRXU+/RYNeuXczLy4uZm5uzHj16sP/85z+838Fjx44xf39/JpfLmUQiYU8++SRLSEjg9vfo0aPJczf1+377VYI7xb927Vrm7u7OzM3NmZubG1uxYgW7efNms21fU1PD/vOf/zAnJycmFouZg4MDmzp1Kq/M3era1OfG3Y5hjLH169czDw8P7u94woQJjLH639nb2yYzM5P7/frll1+4c6SkpLBRo0Yxa2trZm1tzcaMGcPS0tK4/Q3tGR8fzx599FFmaWnJfHx82JkzZ5ptk3tBScjAWpqEJBIJe/vtt1lqairbvHkzA8BGjhzJ3nrrLZaamsq2bdvGALDDhw9zx02aNIn16NGDHTlyhCUnJ7P58+ezLl26sMuXLzPGGDt//jwDwPbu3cuuXr3K8vPzGWON/ygvXrzITE1N2WuvvcYuX77MDh8+zFxcXNiMGTN49ZBIJOz5559nly5dYr/++ivr2bMnr8zYsWPZ8OHD2R9//MEyMzPZTz/9xL7++utm652bm8usrKzYiy++yJKSktixY8dY//7975iEzp49y0xNTdn27dtZVlYW02g0bNOmTezKlSusurqarV+/ngFgV69eZVevXmU6nY5rYxsbGzZnzhyWlJTENBoNt/32JGRra8tmz57NkpOT2cGDB5mdnR177bXXeGXulITKysrYtGnT2L/+9S8ujoZLTrcnobv9DBs+JHr16sV2797N0tLS2IIFC5ipqSlLTU1ttm2vXr3KIiIi2Llz51hGRgaLjo5m1tbWbMuWLVyMe/fuZQDY+fPnubYaMmQImzJlChd3dXU1q6ioYB4eHmzChAns999/Z2lpaWzFihVMLBaz5ORkxtj/PsTd3d3ZwYMHWUZGBrty5UqjuHQ6HVu9ejUzNTXl3oOx+t9JmUzGduzYwdLT09mJEydY//79eb9f+/btY7t372YpKSksMTGRhYSEMLlczgoLCxlj9Zc6TU1N2erVqxudu6VJ6Pb4lyxZwpRKJdu3bx/LyMhg33//PXNxcWHvvfdes23/8ccfMycnJ/bzzz+zv/76i505c4Z9+umn3P6W1PX2z42WHLN48WJmbW3N1q1bx1JTU9m5c+fYihUrGGP1l5179uzJ3njjDa5tamtrGyWhiooKplQq2bBhw9jZs2fZ2bNn2dChQ1nv3r1ZdXU1F4tIJGJDhgxhJ0+eZJcvX2YjR45kPXv2vGNybilKQgY2a9YsZmpqyn2ruPXbRQN/f3/m5eXFO+7hhx9m/fr1423z9PRkb7zxBmOMsbS0NAaAff/997wyjz76KAsKCmKMNT8mdPsf5YwZM9hjjz3GK3PgwAEmEolYVlYWVw87OztWVVXFlYmMjGQODg68+G6/5nwnCxcuZEqlkveLe+jQoTsmoX379jGJRMIbx7lVc2NC/v7+rE+fPqyurq7R9tuTUI8ePVhtbS237YsvvmDm5ubct867JSHGmh8TujUJteRn2PAh8fHHH3P7a2trmY2NDdu4cWOTbdCc+fPns4CAAO51wwfvrcmiqTGhrVu3Micnp0YfME899RQLDw/nnWvHjh13jaOppNCjRw/2+eef87adOHGCAWB///13k+epq6tjMpmM7dy5k9vW1JjQvSShW+O/ceMGs7S0ZD/88APv2O3btzOpVNps/ebPn8+eeuoprtd5u5bU9fYkdLdjysvLmYWFBfvwww+bjaupMaHbk9CXX37JLC0tWUFBAVfm2rVrzMLCgm3fvp0xVt+eANi5c+e4Mg3jfCkpKc2+f0vRmFArePzxx7F9+3beNisrK95rLy8v3msHB4dG12odHByQn58PAEhOTgYAPPnkk7wyTz75JH777bd7ii8pKQnDhg3jbfP39wdjDMnJyejRowcAoG/fvjA3N+fKODo64vr169zr1157DXPmzMEPP/yAoUOHYvTo0Y3iu1VycjIGDRrEm6nn5+d3x1gDAwPh6uqKXr16ITAwEMOGDcOECROgUCjuWs8BAwa0aCxu0KBBMDU15V4PHjwY1dXVSE9Ph6en512Pb6l7+Rl6e3tz/zc1NYW9vT2v7W+n1+vxwQcf4JtvvkFOTg6qqqpw8+ZN7md5L37//Xdcu3YNMpmMt726urrR2OKgQYPu+fwFBQX466+/8Prrr+PNN9/ktrN/7qX8559/4rHHHkNmZiYWL16M3377Dfn5+dDr9aioqMBff/11z+/ZnFvjT0pKQmVlJSZOnAiRSMRtr6urQ1VVFQoKCpoc4wsKCkJgYCDc3NwQGBiIwMBAjB07FmKxuMV1vVVLjmGMoaqqCk8//fQD1T8pKQkPP/ww7++pW7ducHd3R1JSErdNJBLxPrMcHR0BANevX4e7u/sDxUBJqBVYWlrCzc3tjmVuHwAViURNbtPr9QaPr6XEYjHvtUgk4v4QgPo/vpEjR+LIkSP4+eef8cwzz+C5554z6PRqGxsbnD17FqdOnYJarcbGjRvx9ttvIy4uDgMGDLjjsdbW1gaJwcTEhFdvoH4QuzU11fZ3+l34+OOPERERgU8//RSPPvoobG1t8emnn+L777+/5/fW6/Xw8PDA/v37G+27/cvU/bRxQz3WrFmDp556qtF+Z2dnAMCYMWOgUCjw2WefwcXFBWKxGH5+fqipqbnj+Zv64tHcz+vW+Bvi2rNnD1QqVaOyzU1q8fb2RmZmJn788Uf8/PPPCA8Px6JFi3D69OkW1/VWLTlGo9E0GUtrMTEx4X1Ra0jShvh8onVCRuKRRx4BAJw8eZK3/eTJk+jXrx+A/31w1dXV3fVct5/nxIkTEIlE3Pu0VPfu3REUFIQdO3Zg8+bN+Oqrr1BaWtpk2YcffhhnzpzhxXfq1Km7voepqSmefPJJLFu2DOfOnUP37t3x9ddfA2h5ne/k999/5x3/66+/wtzcHL179wYA2NvbIy8vj3fM+fPnea/FYnGL2h2488/wfp08eRIjR45EcHAwHn30Ubi5uSEtLe2uxzUV98CBA5GRkQGJRAI3Nzfev4ZvwA+iW7ducHFxQWpqaqPzu7m5wcLCAkVFRUhOTsY777yDESNG4OGHH4aFhQV3ZeBO8dvb26Ouro7Xc7z959WURx55BBYWFsjIyGgyrls/hG9nY2OD5557DmvXrsXZs2dx+fJlnDhxokV1vZ/2aWiPY8eONRtTS38nk5OTUVhYyG27fv06UlNTH/h3sqWoJ9QKampqcO3atUbbGxbr3Y/evXtj8uTJmDt3Lr744gv06NEDn3/+ORITE7kPZIVCARsbGxw7dgyPPPIIzM3NIZfLG53rrbfego+PD/7zn/9gzpw5yMrKwquvvorp06dDqVS2OKawsDCMGjUK7u7uqKqqwr59++Di4gJbW9smy7/yyiv45JNPEBoaijfffBN5eXlYuHDhHd8jJiYGGRkZePLJJ2FnZ4dz587hypUrePjhhwEAvXr1AgAcPHgQfn5+sLS0hI2NTYvrAABFRUWYN28ewsPDkZGRgUWLFmHOnDnct+SAgAC88sor2LNnDx599FF89913+OWXX3iXq3r16oU9e/YgKSkJ3bp1g62tLe9SJtCyn+H9cnd3R3R0NH7++Wc4OTlhx44dSEhIaPLnf6tevXrh559/Rnp6OqRSKaRSKaZPn45PP/0Uo0ePxsqVK6FSqXD9+nX89NNP8PDwwPjx4x8oVgBYuXIlQkJCIJfLMW7cOHTp0gWXL1/GDz/8gC+++AJyuRx2dnbYtGkTevfujaKiIrz99tuNLgc2xP/MM89ALBZDoVBg0KBBsLW1xTvvvIN3330X6enpWLZs2V1jsrGxwbvvvot3330XIpEIAQEBqK2txaVLl/DHH39g1apVTR734YcfwtHREd7e3rCyssKuXbtgamrK9abuVtf7aR8bGxu88cYbeP/992FpaYnAwEBUVlbi8OHDWLBgAdc2p06dQnZ2NqysrJrsyU2bNg3Lli3D1KlT8eGHH4IxhjfffBNOTk6YOnXqXdvMIB54VInwNLdYFQA3+NfUQHdTA8QjRoxg06dP516XlJTccXovY/WDqD179mSmpqYtnqKtUCjYyy+/fNcpo7dPApg7dy7r06cPs7CwYF27dmWjRo1iiYmJd2wftVrN+vXrx8RiMXvkkUdYXFzcHScmnDhxgj311FNMoVBwU2YjIiJ45wwPD2d2dnaNpmg3tQi1uSnab775JuvatSuzsbFhISEhvAWVNTU13HtIpVI2d+5ctmjRIt7EhKKiIvbMM88wiURyxynad/sZNjWFlrGmB5lvpdPp2OTJk5mtrS3r2rUrmzt3Lnvvvfd4MTY1MSE9PZ0NGTKEWVtb8wbuCwsL2csvv8wcHR1Zly5dmKOjIxs/fjw7f/58s+dqTlO/f4wxtn//fubr68ssLS2Zra0t8/LyYkuXLuX2Hz9+nHl6ejJzc3OmUqnYd99916gdfvjhB9a3b1/WpUsX3u9mbGws69u3L7OwsGBPPPEEO3LkSJMTE5qKf9OmTdzUaJlMxgYNGtTk9PMGGzduZD4+PszW1pZZW1uzgQMHsgMHDtxTXZv6e7vbMXq9nq1evZqpVCrWpUsXZm9vzyZNmsTt//3339mjjz7KLCws7jpF+5lnnuEmUY0ePbrJKdq3am4S1P0QMUZPViWEECIMGhMihBAiGEpChBBCBNNmExOSk5Nx6NAhZGRkoLi4GHPnzsXQoUN5ZfLy8vD1118jMTERtbW1cHJywquvvspNY7x58yaio6Nx6tQp1NTUoF+/fpg9ezYeeugh7hyFhYX48ssvkZSUBLFYjMGDB2PmzJm8tSnJycnYvn07cnJyIJfL8eyzzz7wfHtCCCH3rs2SUFVVFVxcXODv74/169c32p+fn49FixbB398fixcvhrW1NXJzc3lTGLdt24azZ88iPDwcNjY22LFjByIjI7Fq1SqYmJhAr9cjIiICtra2WLp0KcrLy/HZZ58BAIKDg7n3iYiIwFNPPYVXX30VKSkp2Lx5MyQSCXx9fdumMQghhABow8txPj4+mDZtGnx9fZucprxr1y54eXlh5syZcHV1Rbdu3eDj48Ot5K2oqMBPP/2EGTNmwNPTE66urggLC0N2dja3cOvixYvIyclBWFgYXF1d4enpienTpyMuLg4VFRUAgGPHjkEulyM4OBjOzs4ICAiAv78/Dh061FZNQQgh5B/tYkxIr9fj3LlzcHZ25ubHL1iwAL/++itXJiMjA3V1dbxbRygUCjg5OUGr1QIAtFotnJyceLeg8Pb2xs2bN5GRkQGg/rbyt9+KxcvLCxkZGaitrW3NahJCCLlNu1isWlpaiqqqKuzfvx9Tp07F9OnTkZiYiLVr18LCwgI+Pj7Q6XQwMTFptBBSKpVCp9MBAHQ6HaRSKW+/ra0tTExMeGX69+/f6Bx1dXUoKytrtLhPrVZDrVYDACIjI+96y5D2wMzMjBKqAVF7Gg61pWEZS3vefhuqW7WLJNRw/6GBAwdizJgxAOofi5yeno4jR47Ax8dHsNgCAgJ4j9299fYW7ZVCoTCKOI0FtafhUFsalrG0551u99QuLsdJJBKYmpo2upmfk5MT18AymQx6vR5lZWW8MiUlJdztU2QyGUpKSnj7y8rKoNfreWUaekW3nsPU1LTZ280QQghpHe0iCZmZmaF3796NbhJ59epV7tbprq6uMDU15d09tqioCLm5udw9mlQqFXJzc1FUVMSV0Wg06NKlC1xdXQEAffr0waVLl3jvo9Fo4OrqypvGTQghpPW1WRKqqqpCVlYWsrKywBhDYWEhsrKyuJ7Os88+i19//RVqtRrXrl2DWq3Gr7/+ihEjRgCov4X8sGHDsHPnTmg0GmRmZmL9+vVQKpXcRAMvLy84Oztj/fr1yMzMhEajQXR0NIYPH87dgv7pp5/G33//jW3btiEnJwdxcXE4fvw4xo4d21ZNQQgh5B9tdu+4pKQkLF26tNF2f39/zJs3DwBw/Phx7N+/H4WFhejevTvGjx/Pe+hZw2LV+Ph43mLVW2fDNSxWTUxM5J4/8sILL/Ce1dOwWPXKlSvcXWpbulj19t5ae2Qs14mNBbWn4VBbGpaxtOedxoToBqb3iJJQ50PtaTjUloZlLO3Z7icmEEII6ZwoCRFCCBEMTQcTmNMmJ6FDaJHcl3KFDoEQ0gFRT4gQQohgKAkRQggRDCUhQgghgqEkRAghRDCUhAghhAiGkhAhhBDBUBIihBAiGEpChBBCBENJiBBCiGAoCRFCCBEMJSFCCCGCoSRECCFEMJSECCGECKbN7qKdnJyMQ4cOISMjA8XFxZg7dy6GDh3aZNmoqCio1WrMmDEDzz77LLe94cmqp06d4j1Z9aGHHuLKNDxZNSkpCWKxGIMHD8bMmTNhZva/qjY8WTUnJwdyuRzPPvtsi5+sSgghxHDarCdUVVUFFxcXBAUFQSwWN1vu9OnT+PPPPyGXyxvt27ZtGxISEhAeHo6lS5eisrISkZGR0Ov1AAC9Xo+IiAhUVVVh6dKlCA8PR0JCAnbs2MGdIz8/HxEREXB3d8eqVaswfvx4bN26FadPnzZ8pQkhhNxRmyUhHx8fTJs2Db6+vhCJRE2WKSgowNatWzF//nxezwUAKioq8NNPP2HGjBnw9PSEq6srwsLCkJ2dDY1GAwC4ePEicnJyEBYWBldXV3h6emL69OmIi4tDRUUFAODYsWOQy+UIDg6Gs7MzAgIC4O/vj0OHDrVuAxBCCGmk3YwJ1dXVYc2aNZg4cSKcnZ0b7c/IyEBdXR28vLy4bQqFAk5OTtBqtQAArVYLJycnKBQKroy3tzdu3ryJjIwMAEBaWho8PT155/by8kJGRgZqa2tbo2qEEEKa0W6erPrtt9/C1ta22bEZnU4HExMT2Nra8rZLpVLodDqujFQq5e23tbWFiYkJr0z//v0bnaOurg5lZWWNLgOq1Wqo1WoAQGRkJC/BdSadtd4AYGZm1qnrb0jUlobVEdqzXSShpKQkHD9+HB9++KHQoTQSEBCAgIAA7nVhYaGA0Qins9YbqE/Anbn+hkRtaVjG0p6Ojo7N7ms3SUin0yE0NJTbptfr8dVXX+Hw4cPYuHEjZDIZ9Ho9ysrKIJFIuHIlJSXw8PAAAMhkMqSmpvLOXVZWBr1eD5lMxpVp6BXdeg5TU9NGvSxCCCGtq10koREjRsDX15e3beXKlRg8eDDXC3F1dYWpqSk0Gg38/PwAAEVFRcjNzYVKpQIAqFQq7Nu3D0VFRdy0bY1Ggy5dusDV1RUA0KdPH/z++++899JoNHB1dW00GYIQQkjrarNP3aqqKly7dg0AwBhDYWEhsrKyYGNjA4VC0Wgsx8zMDDKZjOvGWVlZYdiwYdi5cyckEglsbW2xY8cOKJVKbqKBl5cXnJ2dsX79esycORNlZWWIjo7G8OHDYWVlBQB4+umncfToUWzbtg0BAQFITU3F8ePHER4e3lZNQQgh5B9tloTS09OxdOlS7vW3336Lb7/9Fv7+/pg3b16LzvHiiy/C1NQUq1ev5harzps3DyYm9ZP8TExMsGDBAnz55ZdYtGgRxGIx/Pz88MILL3DnsLe3x4IFC7B9+3ZuunZQUFCjnhghhJDWJ2KMMaGDMCZ5eXkGPZ/TJieDnq+15L6UK3QIgjGWwV9jQG1pWMbSnneamNBu1gkRQgjpfCgJEUIIEQwlIUIIIYKhJEQIIUQwlIQIIYQIhpIQIYQQwVASIoQQIhhKQoQQQgRDSYgQQohgKAkRQggRDCUhQgghgqEkRAghRDCUhAghhAiGkhAhhBDBUBIihBAiGEpChBBCBNNmT1ZNTk7GoUOHkJGRgeLiYsydOxdDhw4FANTW1uKbb77BhQsXcP36dVhaWuKRRx7B9OnToVAouHPcvHkT0dHROHXqFPdk1dmzZ+Ohhx7iyhQWFuLLL79EUlISxGIxBg8ejJkzZ8LMzIwXy/bt25GTkwO5XI5nn30WTz/9dFs1BSGEkH+0WU+oqqoKLi4uCAoKglgs5u2rqalBZmYmJkyYgFWrVuHtt99GUVERVq5cibq6Oq7ctm3bkJCQgPDwcCxduhSVlZWIjIyEXq8HAOj1ekRERKCqqgpLly5FeHg4EhISsGPHDu4c+fn5iIiIgLu7O1atWoXx48dj69atOH36dNs0BCGEEE6bJSEfHx9MmzYNvr6+EIlEvH1WVlZYtGgRnnjiCTg6OsLNzQ2hoaHIzc1Fbm79Y6UrKirw008/YcaMGfD09ISrqyvCwsKQnZ0NjUYDALh48SJycnIQFhYGV1dXeHp6Yvr06YiLi0NFRQUA4NixY5DL5QgODoazszMCAgLg7++PQ4cOtVVTEEII+Ue7HRNqSBrW1tYAgIyMDNTV1cHLy4sro1Ao4OTkBK1WCwDQarVwcnLiXcLz9vbGzZs3kZGRAQBIS0uDp6cn7728vLyQkZGB2traVq0TIYQQvjYbE7oXtbW1iI6OxoABA7jxHp1OBxMTE9ja2vLKSqVS6HQ6roxUKuXtt7W1hYmJCa9M//79G52jrq4OZWVlkMvlvH1qtRpqtRoAEBkZyUtwnUlnrTcAmJmZder6GxK1pWF1hPZsd0morq4Oa9euxY0bN/D2228LHQ4CAgIQEBDAvS4sLBQwGuF01noD9Qm4M9ffkKgtDctY2tPR0bHZfe3qclxdXR3WrFmD7OxsLF68mNfrkclk0Ov1KCsr4x1TUlICmUzGlSkpKeHtLysrg16v55Vp6BXdeg5TU9NGvSxCCCGtq90kodraWnz66af466+/sGTJEi5pNHB1dYWpqSk3CQEAioqKkJubC5VKBQBQqVTIzc1FUVERV0aj0aBLly5wdXUFAPTp0weXLl3inVuj0cDV1ZU3jZsQQkjra9Mp2llZWcjKygJjDIWFhcjKykJhYSHq6urwySefIC0tDeHh4RCJRNDpdNDpdKipqQFQP4Nu2LBh2LlzJzQaDTIzM7F+/XoolUpuooGXlxecnZ2xfv16ZGZmQqPRIDo6GsOHD4eVlRUA4Omnn8bff/+Nbdu2IScnB3FxcTh+/DjGjh3bVk1BCCHkHyLGGGuLN0pKSsLSpUsbbff398fkyZMRFhbW5HG3LmptWKwaHx/PW6x668Bcw2LVxMREiMVi+Pn54YUXXkCXLl24Mg2LVa9cuQK5XI5x48a1eLFqXl7ePdT67pw2ORn0fK0l96VcoUMQjLFcdzcG1JaGZSzteacxoTZLQh0FJaHOx1j+0I0BtaVhGUt7Gs3EBEIIIZ0LJSFCCCGCoSRECCFEMJSECCGECIaSECGEEMFQEiKEECIYSkKEEEIEQ0mIEEKIYCgJEUIIEQwlIUIIIYKhJEQIIUQwlIQIIYQIhpIQIYQQwVASIoQQIhhKQoQQQgRDSYgQQohgzNrqjZKTk3Ho0CFkZGSguLiY98RUAGCMYc+ePYiLi0N5eTn69OmDkJAQuLi4cGXKy8uxdetWnD17FgAwcOBABAcHw9ramiuTnZ2NzZs3488//4SNjQ0CAwMxceJEiEQirszp06exe/duXL9+Hd26dcO///1vDBo0qPUbgRBCCE+b9YSqqqrg4uKCoKAgiMXiRvtjYmIQGxuLoKAgREREQCKRYMWKFaisrOTKrF27FpmZmVi4cCEWLlyIzMxMrFu3jttfUVGB5cuXQyqVIiIiAkZflJ0AACAASURBVEFBQTh48CBiY2O5MlqtFqtXr8aQIUPwwQcfYMiQIfjkk0+QlpbWug1ACCGkkTZLQj4+Ppg2bRp8fX15vRKgvhd0+PBhjB8/Hr6+vlAqlQgLC0NlZSXi4+MBADk5Obhw4QJCQ0OhUqmgUqkQGhqK8+fPc4/cjo+PR01NDcLCwqBUKuHr64tx48YhNjYWDU8x//777/HII49gwoQJcHZ2xoQJE/DII4/g+++/b6umIIQQ8o92MSaUn58PnU4HT09PbptYLIaHhwdSU1MB1PdgLCws4O7uzpVxd3eHubk5r0zfvn15PS0vLy8UFxejoKCAK+Pl5cV7fy8vL2i12larHyGEkKa12ZjQneh0OgCATCbjbZdKpSguLubKSCQSXi9KJBJBKpVyx+t0OnTt2pV3joZz6nQ62NvbQ6fTQSqVNnqfhnPcTq1WQ61WAwAiIyOhUCjut5pGrbPWGwDMzMw6df0NidrSsDpCe7aLJNSeBQQEICAggHtdWFgoYDTC6az1BuoTcGeuvyFRWxqWsbSno6Njs/vaxeW4W3srtyopKeF6LTKZDKWlpdzYDlA/llRSUsIdL5PJUFJSwjvH7b2spsrceg5CCCFtp10kIXt7e8hkMmg0Gm5bTU0NUlJSuDEglUqFqqoq3tiNVqtFdXU1r0xKSgpqamq4MhqNBnK5HHZ2dlyZW9+noYxKpWq1+hFCCGlam07RzsrKQlZWFhhjKCwsRFZWFgoLCyESiTBq1CjExMQgISEB2dnZ2LBhAywsLODn5wcAcHZ2hre3N6KioqDVaqHVahEVFQUfHx+uq+fn5wexWIwNGzYgOzsbCQkJiImJwZgxY7ixpFGjRiExMREHDhxAbm4u9u/fj6SkJIwePbqtmoIQQsg/ROzW61utKCkpCUuXLm203d/fH/PmzeMWq6rVaty4cQNubm4ICQmBUqnkypaXl2PLli04d+4cAGDAgAEICQlpdrGqtbU1AgMDMWnSpEaLVb/55htcv34dDg4OeP755/H444+3qB4N08ENxWmTk0HP11pyX8oVOgTBGMt1d2NAbWlYxtKedxoTarMk1FFQEup8jOUP3RhQWxqWsbRnu5+YQAghpHO67ySUmJiI5ORkQ8ZCCCGkk2lxElqyZAlSUlIAAAcOHMCaNWuwZs0a7Nu3r9WCI4QQ0rG1OAlduXKFm8YcFxeHJUuWYOXKlfjxxx9bLThCCCEdW4vvmNAwf+HatWsA6qdMA8CNGzdaISxCCCGdQYuTkLu7O7Zs2YLi4mI89thjAOoTkq2tbasFRwghpGNr8eW4efPmwcrKCj169MCUKVMA1E9XHjVqVKsFRwghpGNrcU8oMTER06ZN423z8fHB6dOnDR4UIYSQzqHFPaGNGzc2uf2LL74wWDCEEEI6l7v2hK5fvw4A0Ov1yM/P593F+vr1600+qpsQQghpibsmofnz53P/f/XVV3n7ZDIZJk+ebPioCCGEdAp3TUK7d+8GUL9YtakbkBLSntC9+AgxLi0eE6IERAghxNBaPDsuPz8fu3btQlZWFqqqqnj7Pv/8c4MHRgghpONrcRJas2YNunXrhpkzZ8Lc3Lw1YyKEENJJtDgJ5eTkYPny5TAxoac/EEIIMYwWJyEPDw9kZWXB1dW1VQLR6/X49ttv8csvv0Cn00Emk2HIkCGYPHkyTE1NAYB7+mpcXBzKy8vRp08fhISEwMXFhTtPeXk5tm7dirNnzwIABg4ciODg4GafvmpjY4PAwEBMnDiR9/RVQgghra/FScjOzg4rV67EoEGDIJPJePumTp36wIEcOHAAR48exbx586BUKpGdnY3PPvsMZmZmmDRpEgAgJiYGsbGxmDt3LhwdHfHdd99hxYoVWL16NSwtLQEAa9euRWFhIRYuXAigfpHtunXr8M477wAAKioqsHz5cnh4eCAiIgJ5eXnYsGEDzM3NMXbs2AeuByGEkJZr8bW16upqDBgwAHV1dSgqKuL9MwStVosBAwZg4MCBsLe3x8CBAzFgwAD8+eefAOp7QYcPH8b48ePh6+sLpVKJsLAwVFZWIj4+HkD9JcMLFy4gNDQUKpUKKpUKoaGhOH/+PPdY7vj4eNTU1CAsLAxKpRK+vr4YN24cYmNjQU86J4SQttXintDcuXNbMw707dsXR48eRW5uLpycnJCTk4OkpCSMHz8eQP3sPJ1OB09PT+4YsVgMDw8PpKamIjAwEFqtFhYWFnB3d+fKuLu7w9zcHKmpqXB0dIRWq0Xfvn15d3rw8vLC7t27UVBQAHt7+1atJyGEkP9pcRJquH1PU7p16/bAgYwbNw6VlZV4/fXXYWJigrq6OkyYMAEjRowAAOh0OgBodClQKpWiuLiYKyORSHhjOyKRCFKplDtep9Oha9euvHM0nFOn0zVKQmq1Gmq1GgAQGRkJhULxwHU1Rp213q2ls7anmZlZp617a+gI7dniJHTr7Xtu13BXhQfx66+/4uTJk5g/fz5cXFyQlZWFrVu3wt7eHsOGDXvg89+vgIAABAQEcK8LCwsFi0VInbXeraWztqdCoei0dW8NxtKejo6Oze5rcRK6PdHodDrs2bMHHh4e9x/ZLXbu3ImxY8di8ODBAAClUomCggLs378fw4YN4/VWbs38JSUlkEqlAOp7NKWlpWCMcb0hxhhKSkq442UyGUpKShrVpWEfIYSQtnPfi35kMhlefPFFfP311wYJpLq6utEaJBMTE26ygL29PWQyGTQaDbe/pqYGKSkp3BiQSqVCVVUVtFotV0ar1aK6uppXJiUlBTU1NVwZjUYDuVwOOzs7g9SFEEJIyzzQytO8vDxUV1cbJJABAwbgwIEDOH/+PPLz83HmzBnExsZi0KBBAOrHdkaNGoWYmBgkJCQgOzsbGzZsgIWFBfz8/AAAzs7O8Pb2RlRUFLRaLbRaLaKiouDj48N1B/38/CAWi7FhwwZkZ2cjISEBMTExGDNmDK0TIoSQNiZiLZyXvHjxYt6HdHV1Na5cuYJJkybhueeee+BAKisrsXv3bpw5cwYlJSWQy+V44oknMGnSJG4mW8NiVbVajRs3bsDNzQ0hISFQKpXcecrLy7FlyxacO3cOQH1yCwkJaXaxqrW1NQIDAzFp0qQWJaGGqd6GQnd9Nixqz/bNWMYwjIWxtOedxoRanISOHz/Oe21hYYEePXqge/fuDxScsaEk1L5Re7ZvxvKhaSyMpT0NMjFh6NChhoiFEEII4bQ4CdXW1mLfvn04efIkiouLIZfL8eSTT2LChAkwM2vxaQghhBBOi7PHzp07kZ6ejpdeegl2dnYoKCjA3r17UVFRgRdffLEVQySEENJRtTgJnT59Gh9++CFsbW0B1F/j69WrF9566y1KQoQQQu5Li6do0809CSGEGFqLe0L/+te/sGrVKkyaNImbkbF37174+vq2ZnyEEEI6sBYnoRkzZmDv3r3YvHkziouL0bVrVwwePBgTJ05szfgIIYR0YHdNQikpKTh79ixmzJiBqVOn8h5gt3PnTmRkZEClUrVqkIQQQjqmu44J7d+/Hw8//HCT+/r164d9+/YZPChCCCGdw12TUFZWFry9vZvc179/f2RmZho8KEIIIZ3DXZNQZWUlamtrm9xXV1eHyspKgwdFCCGkc7hrEnJycsLFixeb3Hfx4kU4ORnHvboIIYS0P3dNQqNHj0ZUVBQSEhKg1+sBAHq9HgkJCdi0aRNGjx7d6kESQgjpmO46O87Pzw86nQ6fffYZbt68CYlEgtLSUnTp0gVTpkzhnuVDCCGE3KsWrRMaM2YMhg0bBq1Wi/LyctjY2EClUsHKyqq14yOEENKBtXixqpWVVbOz5AghhJD70a6ewVBcXIyvvvoKf/zxB6qqqmBvb4+XXnqJW6fU8GTVuLg4lJeXo0+fPggJCYGLiwt3jvLycmzduhVnz54FAAwcOBDBwcHNPlnVxsYGgYGBmDhxIj3emxBC2liLb2Da2m7cuIFFixYBABYsWIBPP/0UwcHBkEgkXJmYmBjExsYiKCgIERERkEgkWLFiBW+a+Nq1a5GZmYmFCxdi4cKFyMzMxLp167j9FRUVWL58OaRSKSIiIhAUFISDBw8iNja27SpLCCEEQDvqCcXExEAulyMsLIzbZm9vz/2fMYbDhw9j/Pjx3E1Tw8LCMHv2bMTHxyMwMBA5OTm4cOECli1bxt1KKDQ0FIsXL0ZeXh4cHR0RHx+PmpoahIWFQSwWQ6lUIjc3F7GxsRgzZgz1hgi5hTE8Lr2zPiq9o2g3PaHff/8dbm5u+PTTTzF79my89dZbOHLkCPcIifz8fOh0Onh6enLHiMVieHh4IDU1FQCg1WphYWEBd3d3roy7uzvMzc15Zfr27QuxWMyV8fLyQnFxMQoKCtqiqoQQQv7RbnpC+fn5OHbsGEaPHo3x48cjKysLW7ZsAQCMHDkSOp0OACCTyXjHSaVSFBcXAwB0Oh0kEgmvNyMSiSCVSrnjdTodunbtyjtHwzl1Oh2v9wUAarUaarUaABAZGQmFQmGoKhuVzlrv1kLtaTiduS3NzMyMvv7tJgnp9Xr07t0b06ZNAwD06tULV69exdGjRzFy5EjB4goICEBAQAD3urCwULBYhNRZ691aqD0NpzO3ZcOz3do7R0fHZve1m8txcrkczs7OvG3Ozs5cA9/aW7lVSUkJpFIpV6a0tJT3FFjGGEpKSrjjZTIZSkpKeOdorpdFCCGkdbWbJOTu7o68vDzetry8PK6raW9vD5lMBo1Gw+2vqalBSkoKNwakUqlQVVUFrVbLldFqtaiuruaVSUlJQU1NDVdGo9FALpfDzs6u1epHCCGksXaThEaPHo20tDTs27cP165dw2+//YYffvgBI0aMAFA/tjNq1CjExMQgISEB2dnZ2LBhAywsLLhbBzk7O8Pb2xtRUVHQarXQarWIioqCj48P1x308/ODWCzGhg0bkJ2djYSEBMTExNDMOEIIEYCI3XrtSmDnz5/Hrl27uB7QiBEj8Mwzz3DJoWGxqlqtxo0bN+Dm5oaQkBAolUruHOXl5diyZQvOnTsHABgwYABCQkKaXaxqbW2NwMBATJo0qUVJ6Pbe2oMyhimwgPFMg6X2NCxjaE9jacvW0BHGhNpVEjIGlITaN2pPwzKG9jSWtmwNHSEJtZvLcYQQQjofSkKEEEIEQ0mIEEKIYCgJEUIIEQwlIUIIIYKhJEQIIUQwlIQIIYQIhpIQIYQQwVASIoQQIhhKQoQQQgRDSYgQQohgKAkRQggRDCUhQgghgqEkRAghRDCUhAghhAiGkhAhhBDBmAkdQHP279+PXbt2YcSIEQgJCQHwvyerxsXFoby8HH369EFISAhcXFy448rLy7F161acPXsWADBw4EAEBwc3+2RVGxsbBAYGYuLEifR4b0IIaWPtsiek1WqhVqvRo0cP3vaYmBjExsYiKCgIERERkEgkWLFiBSorK7kya9euRWZmJhYuXIiFCxciMzMT69at4/ZXVFRg+fLlkEqliIiIQFBQEA4ePIjY2Ng2qx8hhJB67S4JVVRUYN26dXjllVd4vRfGGA4fPozx48fD19cXSqUSYWFhqKysRHx8PAAgJycHFy5cQGhoKFQqFVQqFUJDQ3H+/Hnusdzx8fGoqalBWFgYlEolfH19MW7cOMTGxoKedE4IIW2r3SWhL774Ao8//jj69evH256fnw+dTgdPT09um1gshoeHB1JTUwHU96AsLCzg7u7OlXF3d4e5uTmvTN++fSEWi7kyXl5eKC4uRkFBQWtWjRBCyG3a1ZiQWq3GtWvX8Oqrrzbap9PpAAAymYy3XSqVori4mCsjkUh4YzsikQhSqZQ7XqfToWvXrrxzNJxTp9PB3t6+UUxqtRoAEBkZCYVC8SBVNFqdtd6thdrTcDpzW5qZmRl9/dtNEsrLy8OuXbuwfPlymJm1m7AQEBCAgIAA7nVhYaGA0Qins9a7tVB7Gk5nbkuFQmEU9Xd0dGx2X7v5tNdqtSgrK8Prr7/ObdPr9bh8+TJ+/PFHfPLJJwDqeyu3Zv6SkhJIpVIA9T2a0tJSMMa43hBjDCUlJVxvRyaToaSkhPfezfWyCCGEtK52k4Qee+wxfPTRR7xtn3/+ORwcHPDcc8+he/fukMlk0Gg0cHNzAwDU1NQgJSUFM2bMAACoVCpUVVVBq9Vy40JarRbV1dXca5VKha+++go1NTXcuJBGo4FcLoednV1bVZcQQgjaURKytrbmzYYDAHNzc9jY2ECpVAIARo0ahf3798PJyQndu3fHvn37YGFhAT8/PwCAs7MzvL29ERUVhTlz5gAAoqKi4OPjw3UH/fz8sGfPHmzYsAETJkzA1atXERMTg0mTJtE6IUIIaWPtJgm1xLhx41BTU4PNmzfjxo0bcHNzw8KFC2FpacmVmT9/PrZs2YKVK1cCAAYMGMAtdgUAKysrLFq0CJs3b8aCBQtgbW2NMWPGYMyYMW1eH0II6exEjBbH3JOG9UaG4rTJyaDnay25L+UKHUKLUHsaljG0p7G0ZWvoCBMT2t06IUIIIZ0HJSFCCCGCoSRECCFEMJSECCGECIaSECGEEMFQEiKEECIYSkKEEEIEQ0mIEEKIYCgJEUIIEQwlIUIIIYKhJEQIIUQwlIQIIYQIhpIQIYQQwVASIoQQIhhKQoQQQgRDSYgQQohg2s2TVffv348zZ84gLy8PZmZm6NOnD6ZNm8Y92hsAGGPYs2cP4uLiUF5ejj59+iAkJAQuLi5cmfLycmzduhVnz54FAAwcOBDBwcG8R4dnZ2dj8+bN+PPPP2FjY4PAwEBMnDiRHu9NCCFtrN30hJKTk/H0009j+fLlWLJkCUxNTbF8+XKUl5dzZWJiYhAbG4ugoCBERERAIpFgxYoVqKys5MqsXbsWmZmZWLhwIRYuXIjMzEysW7eO219RUYHly5dDKpUiIiICQUFBOHjwIGJjY9u0voQQQtpRElq4cCGeeuopKJVKKJVKvPrqqygtLUVKSgqA+l7Q4cOHMX78ePj6+kKpVCIsLAyVlZWIj48HAOTk5ODChQsIDQ2FSqWCSqVCaGgozp8/zz2WOz4+HjU1NQgLC4NSqYSvry/GjRuH2NhY0JPOCSGkbbWbJHS7yspKMMZgY2MDAMjPz4dOp4OnpydXRiwWw8PDA6mpqQAArVYLCwsLuLu7c2Xc3d1hbm7OK9O3b1+IxWKujJeXF4qLi1FQUNAWVSOEEPKPdjMmdLutW7eiZ8+eUKlUAACdTgcAkMlkvHJSqRTFxcVcGYlEwhvbEYlEkEql3PE6nQ5du3blnaPhnDqdDvb29rx9arUaarUaABAZGQmFQmGoKhqVzlrv1kLtaTiduS3NzMyMvv7tMglt374dqampWLZsGUxMhO2sBQQEICAggHtdWFgoYDTC6az1bi3UnobTmdtSoVAYRf0dHR2b3dfuLsdt27YNp06dwuLFi9GtWzdu+629lVuVlJRAKpVyZUpLS3ljO4wxlJSUcMfLZDKUlJTwztFcL4sQQkjraldJaOvWrVwCcnJy4u2zt7eHTCaDRqPhttXU1CAlJYUbA1KpVKiqqoJWq+XKaLVaVFdX88qkpKSgpqaGK6PRaCCXy2FnZ9ea1SOEEHKbdpOEvvzySxw/fhzh4eGwsbGBTqeDTqdDVVUVgPqxnVGjRiEmJgYJCQnIzs7Ghg0bYGFhAT8/PwCAs7MzvL29ERUVBa1WC61Wi6ioKPj4+HDdQT8/P4jFYmzYsAHZ2dlISEhATEwMxowZQ+uECCGkjYlYO5mXPGXKlCa3T5o0idvXsFhVrVbjxo0bcHNzQ0hICG9Ba3l5ObZs2YJz584BAAYMGICQkJBmF6taW1sjMDAQkyZNalESapjqbShOm5zuXqgdyH0pV+gQWoTa07CMoT2NpS1bQ0cYE2o3SchYUBJq36g9DcsY2tNY2rI1dIQk1G4uxxFCCOl82uUUbUII6WiMoVcJtH3PknpChBBCBENJiBBCiGAoCRFCCBEMJSFCCCGCoSRECCFEMJSECCGECIaSECGEEMFQEiKEECIYSkKEEEIEQ0mIEEKIYCgJEUIIEQwlIUIIIYKhJEQIIUQwlIQIIYQIptM+yuHo0aM4ePAgdDodnJ2d8eKLL8LDw0PosAghpFPplD2hX3/9Fdu2bcNzzz2HVatWwd3dHf/973+N4gmFhBDSkXTKJBQbGwt/f38EBATA2dkZwcHBkMvlOHbsmNChEUJIp9LpklBtbS0yMjLg5eXF2+7p6YnU1FSBoiKEkM6p040JlZaWQq/XQyqV8rbLZDJcunSpUXm1Wg21Wg0AiIyMhKOjo0HjYUuYQc/X2VF7Gha1p+FQWzat0/WE7lVAQAAiIyMRGRkpdCgt9s477wgdQodC7Wk41JaG1RHas9MlIYlEAhMTE5SUlPC263Q6yGQygaIihJDOqdMlITMzM7i6ukKj0fC2X7p0Ce7u7gJFRQghnZPp+++//77QQbQ1S0tLfPvtt5DL5RCLxdi7dy+Sk5Mxd+5cWFtbCx2eQbi6ugodQodC7Wk41JaGZeztKWKMdcrRsobFqsXFxXBxccGsWbPw8MMPCx0WIYR0Kp02CRFCCBFepxsTIoQQ0n5QEiKEECIYSkKEEGIEamtr8e677yIvL0/oUAyKkhAhpNXo9Xro9XrutU6nQ1xcHFJSUgSMyjiZmZkhPz9f6DAMjiYmdADJycno0qUL+vTpAwA4fvw44uLi4OLigpkzZ8LCwkLgCNu/8vLyFpe1sbFpxUg6lv/+97/w9vbGqFGjUFVVhddeew3V1dWoqqrCyy+/DH9/f6FDNCrR0dEAgBdeeEHgSAyn0907riPatm0bJk+eDADIy8tDVFQUhg0bhpSUFERHR+Oll14SOML2LyQkpMVld+/e3YqRdCzp6emYMWMGACAhIQGWlpZYv3494uPjcejQIUpC96i6uhq//PILNBoNXF1dYW5uztsfHBwsUGT3j5JQB3Dt2jUolUoAwOnTp+Hp6YnZs2cjLS0NH3/8MSWhFliyZInQIXRIVVVVsLKyAgBcvHgRgwYNgpmZGfr164fNmzcLHJ3xyc3N5RandpRLc5SEOgCRSMRdd09MTMSgQYMA1N8ZvKysTMjQjAYtVG4dCoUCqampsLGxwcWLF/H6668DqL/8KRaLBY7O+HTEL0uUhDoANzc37N27F56enrh8+TJCQ0MBAAUFBZDL5QJHZ9x0Oh1qa2t52xQKhUDRGJ8xY8Zg/fr1sLCwgEKhgIeHB4D6ccyG3ju5d6Wlpbh+/Tp69uyJLl26CB3OA6Ek1AHMmjULa9euxe+//44JEybAwcEBAPDbb79BpVIJHJ3xqaiowJYtW/Dbb781SkAAjQndi8DAQLi6uqKoqAienp4wMamfkOvg4ICpU6cKHJ3xqaysxOeff46EhAQAwNq1a9GtWzdERUVBJpNhypQpAkd472h2XAdWU1MDExMTmJnRd417sXHjRqSnp2P69On4+OOP8corr+Dvv//G4cOHMXPmTPj6+godolGrra2l38n79OWXX+Kvv/5CSEgIFi1ahI8++gjdunXDuXPn8M033+DDDz8UOsR7RuuEOoDm1mJkZmbSH/t9uHDhAoKDg+Ht7Q0TExO4urpizJgxmDZtGn788UehwzMqhw8fxunTp7nXn3/+OWbMmIHw8PAOt+iyLZw9exazZs1Cz549IRKJuO1OTk64fv26gJHdP0pCHUBkZCSOHDkCoH420jvvvIOdO3fi/fffx4kTJwSOzvjcuHEDdnZ2AAArKytucodKpYJWqxUyNKPzww8/QCKRAKgfB/rtt98wf/589OzZEzt27BA4OuNz48YN2NraNtpeVVXFXeo0NsYZNeFJT09Hv379APxvLcamTZswZ84cHDp0SODojI+DgwP3rdLJyQmnTp0CYwxnzpyhhar36O+//4a9vT0A4Ny5c/D19cUTTzyByZMnIy0tTeDojE/v3r1x9uxZ7nVDb+jHH3802odyUhLqAO60FsNYu+hC8vf3x19//QUAGD9+PNRqNaZNm4bo6GiMGzdO4OiMi5WVFUpLSwEAGo0G/fv3B1B/C5qamhohQzNK//73v7F7925s3LgRdXV1iI2NxdKlS3Hy5EmjnehBAwYdAK3FMKwxY8Zw/+/Xrx9Wr16N9PR0dO/enaYV3yNPT0988cUX6NWrF65du4ZHH30UAHDlyhWuh0Razt3dHStWrMDBgwfh4OCAS5cuoVevXli5cqXR/m5SEuoAaC2G4dTW1mLx4sUICwuDo6MjgPokT2uD7k9ISAi++eYbFBYW4o033uAuZ2ZmZmLw4MECR2eclEolwsLChA7DYGiKdgeRnp7OrcVouGHp+fPnYWVlhb59+wocnXGZPXs2li1bxiUhQtqLwsLCZveJxWJuEogxoSREyG064p2KhaTT6XDy5Elcv34dU6dOhUQiQUpKCrp27UqX5O7R3cZ9rKysMHToUMyYMQOmpqZtFNWDoctxHcTRo0dx9OhR5Ofn4+OPP0a3bt1w4MAB2Nvb44knnhA6PKPSEe9ULJSMjAwsW7YM9vb2uHLlCp599llIJBJoNBpcvXoV4eHhQodoVMLDw7Fz504EBgZyj25JS0uDWq3G5MmTUVFRgb1798LS0tJo7p5ASagD+P7773Hw4EGMGzcOX331Fbe9a9euOHr0KCWhe9QR71QslOjoaIwaNQpTpkzBzJkzue3e3t44fvy4cIEZqWPHjmHWrFl4/PHHuW39+vWDo6MjDh8+jKVLl0IikWDPnj2UhEjb+fHHHzFnzhz4+Pjgm2++4bb36tULV65cETAy49QR71QslIyMDLz88suNtstkMpSUlAgQkXFLS0vDnDlzGm13cXFBeno6gPpF1UVFRW0d2n2jdUIdQEFBAVxcXBptNzU1pbUY92HDhg2orKxstL2qqgobNmwQICLjPENphgAADt1JREFUJRaLcePGjUbb8/LyjHIQXWh2dnZQq9WNtsfFxXEzOEtLS41qUTUloQ6gW7duyMzMbLT9jz/+gLOzswARGbcTJ040mbxrampw8uRJASIyXgMHDsSePXtw8+ZNAPUr/PPz8/HVV1/xLimRlpk5cyaOHDmC119/HZ999hk+++wzvPHGGzhy5Ah3uTM9PR3/+te/BI605ehyXAcwduxYbN68GdXV1WCMQavV4uTJkzh48CBeeeUVocMzGuXl5WiYLHrjxg3e7CK9Xo/z589DKpUKFZ5ReuGFFxAREYHZs2ejuroaixYtQklJCdzd3fH8888LHZ7R8fHxwZo1a3Ds2DHuBrADBgzA008/zfWERowYIWSI94ymaHcQarUa+/bt464Fy+VyTJkyBcOGDRM4MuNxt+mvIpEIU6ZMwYQJE9oooo4jMTERGRkZYIyhV69e8PT0FDok0k5QEupgSktLwRijb+z3ITk5GYwxLFu2jLe6H6i/15lCoUDXrl0FjJCQ+iUEWVlZKCkpwe0f38Z4iZOSECG3KSgogEKh4D2vhdy/tLQ0XLp0CaWlpbznXgG05upeaTQarFmzBuXl5U3uN8an/tKYUAdQXl6OXbt2ITExsclvR9u3bxcoMuNUVlbGPUOoKQ1riMjdHTx4EF999RUcHBwgl8spsT+gbdu2wcfHB//+9787TK+cklAH8PnnnyMrKwvDhw/vML+YQlqwYMEd9xvjt02h/PDDDwgKCsLIkSOFDqVDKCgowNtvv92h/s4pCXUAiYmJeO+997jbeJAHs379et7ruro6ZGZmYt++fZg2bZpAURmniooK7vEN5MG5u7sjLy8PDg4OQodiMJSEOgCJRMLdOZs8uIZHe9/KwcEBVlZW+O677+hD9R4MHjwYFy5cMLppw+1VYGAgoqOjUVxcDKVS2egmpcZ4qZiSUAfQ8LTFsLAwSkatyN7eHllZWUKHYVQUCgW+/fZbpKamQqlUwsyM/5Fz6wMEyd198sknAICoqKgm9xvjpWKaHdcBvPHGGygoKIBer4ednV2jb0cfffSRQJEZp9tnHjHGUFxcjD179iA/Px+rVq0SKDLjM2/evGb3iUSiRpc+yZ0VFBTccX9Tvfj2jnpCHYCvr6/QIXQoISEhTW5/6KGH8Nprr7VxNMbts88+EzqEDsXOzg51dXX4888/UVhYiNraWm6fSCQyyiREPSFCbpOcnMx7LRKJIJFI4ODgYDQPCiMdU25uLlatWoX8/HwwxmBiYgK9Xg9TU1N06dLFKJdjUBIihBjUvdxpfO7cua0YScezcuVKWFtb4+WXX0ZoaCg++OADVFRU4Msvv8Tzzz9vlLdDostxRmrWrFlYt24dJBIJZs6cecdFgMb47Uho2dnZ+PHHH3H9+nW88sorkMvlOHPmDOzs7NCrVy+hw2vXSktLea8vX74MExMTKJVKAPVtyxiDh4eHEOEZtfT0dLz//vuwsLCASCSCXq+Hq6srZsyYgS1bthjl+C8lISMVFBQES0tLAHTrE0O7ePEiPvjgA3h7eyMxMZF7rMP169dx/PhxvP322wJH2L6988473P/3798PsViMuXPncjM3q6qqsHHjRi4pkZZjjHGPm5dIJPj777/h6OiIrl274tq1awJHd38oCRmpoUOHNvn//2/v3mOqrv84jj85XJtHDEMoJUhlyaKsLa1IXBJutdYfMpkyUJYQ/yRm1ho2t9JaFyK7MJWG84iRQzb/cTJjjObZyWNt3abnLFc6hQjNIx2JjnQCPP7+YH5/nR9Y6E/5cr6+HhvbOd9ze8MfvM/n8n5/5P/X3NxMaWkpTzzxRNiR1NnZ2bS0tJgYWeT57LPPePXVV8NKBxISEli6dCmvv/66OpJfpfT0dDo7O0lNTSUzM5N9+/Zhs9lob2+P2AJWHWpnIV6vl9bWVlpbW/F6vWaHE7G6urpGLUi12+1XbBwpowsGg/j9/hHXz58/r1N/r0FBQYHRG7KoqIienh42bdrE0aNHWbVqlcnRXRuNhCzA5/OxefNmOjs7SUpKAjAqql966SVSU1NNjjCy2O12/H4/KSkpYddPnjxpqZ5d4+Hhhx+mrq6OFStWGG2ljh8/zu7du3nooYdMji7yPPDAA8bt1NRUPvjgAwKBAJMmTYrY5rBKQhZQV1dHQkICW7ZsMU5X7OnpYevWrXz88ce89tprJkcYWRYsWMCnn37KunXriIqK4uLFi/zwww80NjaSl5dndngRpaKigk8++YRt27YZNS3R0dE8/vjjrFy50uTorOHv515FIm3RtoCSkhLefPNN7rrrrrDrHR0dbNiwgd27d5sTWIQaGhpi27ZtuN1uYLhO6NKlS+Tm5vLcc8+pVugaBINBzp49Cwx/g1d7KblMIyELSE5OHnV+fWBgwBgZydjFxMTw/PPPs3z5ck6dOkUoFGLmzJnccccdZocWsRISEsjIyDA7DJmANBKygG+//Za9e/eyatUqMjMzAThx4gS7du2ioKCAefPmmRxh5Dl8+DAej2fUQwKrqqpMiioyeb1e3G73iDYzgKaKRUnICkpLSxkcHCQUCmGzDW94vHw7Li4u7LkqXP13jY2NHDhwgOzs7FFPA1WV/9g5nU62b9/O/Pnz+frrr5k3bx5nzpzB5/OxcOHCK/bpk5uHpuMsQMWq15fL5WLt2rVqDHsd7N+/n7KyMvLz8yktLaW4uJjU1FR27NihdSEBlIQsQcWq11coFBqxyUOuzdmzZ7nvvvsAiI2NJRgMAvDkk0+yceNGSkpKzAxPJgAlIYsYGBjg0KFD/PLLLwCkpaWRm5s7YjpO/t3ixYtxuVwsW7bM7FAi3uTJk43EM3XqVLq6usjIyOCPP/5QsaoASkKWcPLkSaqrqxkYGDD6cR08eJA9e/awfv36iDzy10wXLlzA7Xbj8XhGPUJZ059jl5WVxZEjR0hPTycnJ4edO3dy9OhRPB5PRHZ8lutPScgC6uvrmTNnzogmkXV1ddTX1/POO++YHGFk6e7uNqbjTp8+bW4wEa68vNwY8SxZsgSbzcaPP/5ITk4OS5cuNTk6mQiUhCygq6uLysrKEU0iCwsLwzoay9ho2/D109vba+zYtNlszJo1i59//hm73W50gZebmxqYWsCMGTOu2CRy+vTpJkQkMqyuro5Tp04Bw62kampquHDhAm1tbezZs8fk6GQiUBKygKKiInbu3Inb7cbn8+Hz+XC73ezatYuioiICgYDxIzKeuru7jUMAv/rqKzIzM3nllVeorKw02iLJzU3TcRZQXV0NQG1t7YjH3n333bD7zc3N4xKTCAxvd4+JGf434/V6jSMybr/9dnp7e80MTSYIJSEL0BqGTFR33nknbW1tPPjgg3g8HoqLiwHw+/0kJiaaHJ1MBEpCFnDPPfeYHYLIqEpKSqipqWH//v089thjRgnBN998w+zZs02OTiYC9Y6zEL/fP2qTSCUpMVMoFKK/vz/s3Bufz0d8fDxTpkwxMTKZCJSELMDv91NbW8uxY8dGfVzrQCIyUWl3nAU0NDRgs9l4//33iY+PZ9OmTbz44oukpaWxYcMGs8MTEbkirQlZwLFjx1i/fj0zZswgKiqKxMREsrKyiI2Npbm5We1RRGTC0kjIAgYGBpg8eTIwfN58X18fMNzEtLOz08zQRET+kZKQBUyfPt3ocZaRkUFbWxvnzp2jtbWVqVOnmhydiMiVaTrOAp566imj8K+wsJC33noLt9tNTEwMq1evNjk6EZEr0+44C/rrr7/o7u4mOTlZBYEiMqFpJGQRhw8fxuPx8Pvvv/O/3yuqqqpMikpE5J8pCVlAY2MjBw4cIDs7m6SkJKKioswOSURkTJSELMDlcrF27VoeeeQRs0MREbkq2h1nAaFQyDgJVEQkkigJWcDixYtxuVxmhyEictW0Oy5CORwO4/alS5c4dOgQaWlppKenEx0dHfbcsrKy8Q5PRGRMtCYUobq6usLuX56Ou1y0KiISCTQSEhER02hNSERETKMkJCIiplESErG4+vp69u7da3YYIqPSmpDIOFm9ejW9vb3YbP/97rdo0SLKy8uv22c4nU4+//xz3njjjev2niI3knbHiYyjqqoqHTIo8jdKQiImuzx6mT17Nk6nE7vdzpo1azhz5gzNzc0MDg6yYsUKFi1aBEB/fz8Oh4Pvv/+e+Ph48vPzKSgo4PTp02zfvp2hoSFWrlxJdHQ0DQ0NbN26ldtuu42ioiIA2tvb2bdvH4FAgKysLCoqKoxzp5YtW8azzz5LS0sLfX195ObmUl5ern6EcsNoTUhkAjh+/DgZGRk4HA5yc3P58MMPOXHiBLW1taxZswaHw0EwGASGC5X7+/vZsmULGzduxOVy4XQ6SUtLo6KigrvvvpvGxkYaGhpGfI7X66WpqYl169ZRX1/PtGnT+Oijj8Ke89133/H222/z3nvv8eWXX3LkyJHx+BPITUpJSGQc1dTU8Mwzzxg/7e3tAKSkpJCXl4fNZuPRRx/lt99+o7CwkNjYWO6//35iYmL49ddfCYVCuN1uiouLueWWW0hJSeHpp58ec9umL774gry8PGbNmkVsbCzFxcX89NNP+Hw+4zlLlixh0qRJJCcnk52dTUdHx434U4gAmo4TGVcvv/zyiDUhp9PJlClTjPtxcXEA3HrrrWHXgsEgfX19XLx4keTkZOOxadOm4ff7x/T558+fZ+bMmcb9hIQE7HY7fr+flJSUEZ8bHx9vjMBEbgSNhEQiSGJiItHR0fT09BjXenp6jDWdf5OUlBT22mAwSCAQGPPrRa43JSGRCGKz2cjJyaGpqYk///yTc+fO0dLSwsKFC4HhUYzf72doaGjU1y9YsICDBw/S0dHB4OAgTU1NZGZmGqMgkfGm6TiRcVRdXR1WJzR37lzmz59/Ve9RVlaGw+GgsrKSuLg48vPzycvLA+Dee+81NijYbDZ27NgR9tq5c+eyfPlyNm/eTCAQYM6cObzwwgv//y8mco1UrCoiIqbRdJyIiJhGSUhEREyjJCQiIqZREhIREdMoCYmIiGmUhERExDRKQiIiYholIRERMY2SkIiImOY/27j6eVhSpvgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_emotions.emotion.value_counts().plot.bar(color='green', fontsize=14)\n",
    "plt.title(\"Emotions distribution after feature selection\")\n",
    "plt.ylabel(\"Counts\")\n",
    "plt.xlabel(\"Emotion\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jMD2KhVjrett"
   },
   "source": [
    "<h3> 4.4 Verifying the instances per split </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OZHJdYJkkg8Z",
    "outputId": "461b5643-f32c-4826-966f-7682f986f593",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data: 25232\n",
      "Val data: 3154\n",
      "Test data: 3155\n"
     ]
    }
   ],
   "source": [
    "print('Train data: {}'.format(len(X_train)))\n",
    "print('Val data: {}'.format(len(X_val)))\n",
    "print('Test data: {}'.format(len(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DbtbOv5Bqw3_",
    "outputId": "9d5606ab-10a5-47db-a7e9-e6536f80b602"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classes intances in train dataset {'anger': 1146, 'happiness': 12239, 'neutral': 6910, 'sadness': 4937}\n",
      "Classes intances in validation dataset {'anger': 143, 'happiness': 1530, 'neutral': 864, 'sadness': 617}\n",
      "Classes intances in test dataset {'anger': 144, 'happiness': 1530, 'neutral': 864, 'sadness': 617}\n"
     ]
    }
   ],
   "source": [
    "class_count_train = {}\n",
    "class_count_val = {}\n",
    "class_count_test = {}\n",
    "\n",
    "for label in emo_to_label:\n",
    "    class_count_train[label] = np.count_nonzero(y_train == emo_to_label[label])\n",
    "\n",
    "for label in emo_to_label:\n",
    "    class_count_val[label] = np.count_nonzero(y_val == emo_to_label[label])\n",
    "\n",
    "for label in emo_to_label:\n",
    "    class_count_test[label] = np.count_nonzero(y_test == emo_to_label[label])\n",
    "    \n",
    "print('Classes intances in train dataset {}'.format(class_count_train))\n",
    "print('Classes intances in validation dataset {}'.format(class_count_val)) \n",
    "print('Classes intances in test dataset {}'.format(class_count_test))   \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jfjEFVNbeGK5"
   },
   "source": [
    "<h2> 5. Visualizing the size of the tweets </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "id": "GdP-Y2l58AEA"
   },
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased',\n",
    "                                               clean_text=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "id": "nkCaCq72sXXs"
   },
   "outputs": [],
   "source": [
    "len_tokens = list()\n",
    "for txt in df_emotions.content:\n",
    "    # Half of BERT length\n",
    "    tokens = tokenizer.encode(txt, truncation=True, max_length=256) \n",
    "    len_tokens.append(len(tokens))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "e9zdvluyI1pu",
    "outputId": "9bcde457-0a71-4ce4-d56f-365f099c614a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max tokens: 123, min tokens: 3\n"
     ]
    }
   ],
   "source": [
    "print('Max tokens: {}, min tokens: {}'.format(max(len_tokens), min(len_tokens)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 300
    },
    "id": "YDT9PnIjsvTW",
    "outputId": "4923774e-6ec0-413d-c11b-5b0ba7c381cb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Token count')"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEJCAYAAABsc6siAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfXAUdZ7H8XdPZoDEQDKThMQg3hJISljBwIUVUR6EOc8Sz6Uih4uiCyoPRqHAUhfuLN0tTo2bi0GtcLiKuOIeJccB3l5tnXUxQlzjQyCEdUEFRK6kjAzJDIFAQjKZvj+ymU0gIRmSzFM+ryqrenp6un9fOs5n+tcPP8M0TRMREZEAWELdABERiTwKDxERCZjCQ0REAqbwEBGRgCk8REQkYAoPEREJmDXUDQiW77//PtRN6HPJycnU1NSEuhl9LlrrguitLVrrguitrbu60tPTL/t5HXmIiEjAFB4iIhIwhYeIiARM4SEiIgFTeIiISMAUHiIiEjCFh4iIBEzhISIiARswNwkGm2EY/mkNmSIi0Ubh0Q8Mw+Ct/adwNzTjiLWxaGKKAkREoorCo5+4G5qpOdcc6maIiPSLoIRHTU0NxcXFnD59GsMwcDqd3HHHHdTX11NUVMSpU6dISUlh9erVxMfHY5ommzdvZv/+/QwePJi8vDwyMjIA2L17Nzt27AAgNzeXmTNnBqMEERFpJyjhERMTw/33309GRgYNDQ2sWbOGCRMmsHv3bsaPH8/cuXPZtWsXu3btYuHChezfv58ffviBV155hSNHjvDGG2/w/PPPU19fz/bt28nPzwdgzZo15OTkEB8fH4wyRETkL4JytZXdbvcfOcTGxjJixAjcbjcVFRXMmDEDgBkzZlBRUQHA3r17mT59OoZhkJWVxblz5/B4PFRVVTFhwgTi4+OJj49nwoQJVFVVBaMEERFpJ+jnPFwuF99++y1jxoyhrq4Ou90OQGJiInV1dQC43W6Sk5P9n0lKSsLtduN2u0lKSvLPdzgcuN3uTrdTUlJCSUkJAPn5+R3W1998Ph9WqwubDaxWK3a7HYul73PaarUGta5gida6IHpri9a6IHpr621dQQ2PxsZGCgsLWbRoEXFxcR3eMwyjw+WtveV0OnE6nf7XwXwev2EYeL1empub8XrB4/H0y9VWA3WcgUgWrbVFa10QvbVFzHgeXq+XwsJCpk2bxo033ghAQkICHo8HaP2CHTZsGNB6RNG+qNraWhwOBw6Hg9raWv98t9uNw+EIVgkiIvIXQQkP0zTZuHEjI0aM4M477/TPz8nJYc+ePQDs2bOHyZMn++eXlZVhmiaHDx8mLi4Ou91OdnY2Bw4coL6+nvr6eg4cOEB2dnYwShARkXaC0m319ddfU1ZWxrXXXsuTTz4JwIIFC5g7dy5FRUWUlpb6L9UFmDhxIpWVlaxcuZJBgwaRl5cHQHx8PHfffTdr164FYN68ebrSSkQkBAxzgNz6HMwxzA3D4KXy76k510zyVTYen5qucx4BiNa6IHpri9a6IHpri5hzHiIiEj0UHhHM5/P1+VVqIiI9oWdbRSjDMHi17Biusw16+KKIBJ3CI4Lp4YsiEirqthIRkYApPEREJGAKDxERCZjCQ0REAqbwEBGRgCk8REQkYAoPEREJmMJDREQCpvAQEZGAKTxERCRgCg8REQmYwkNERAIWlAcjbtiwgcrKShISEigsLASgqKjIP0DT+fPniYuLo6CgAJfLxerVq/0DkWRmZrJ06VIAjh07RnFxMU1NTUycOJHFixfrceQiIiEQlPCYOXMmt99+O8XFxf55bUPOArz99tvExcX5X6elpVFQUHDJel5//XWWLVtGZmYmL7zwAlVVVUycOLF/Gy8iIpcISrfVuHHjuhxr3DRNPvnkE26++ebLrsPj8dDQ0EBWVhaGYTB9+nQqKir6o7kiItKNkI/n8eWXX5KQkMDVV1/tn+dyuXjqqaeIjY3lZz/7GWPHjsXtdpOUlORfJikpCbfbHYomi4gMeCEPj48//rjDUYfdbmfDhg0MHTqUY8eOUVBQ4D9PEoiSkhJKSkoAyM/PJzk5uc/a3B2fz4fV6sJmA6vVit1ux2Lp24O81iFoT2Gz2fptG6FitVqDur+CKVpri9a6IHpr621dIQ2PlpYWPv/8c/Lz8/3zbDYbNpsNgIyMDFJTU6mursbhcFBbW+tfrra2FofD0eW6nU4nTqfT/7qmpqYfKuicYRh4vV6am5vxelu73Pp6iFjDMDBNs1+3ESrJyclB3V/BFK21RWtdEL21dVdX20VLXQnpT9UvvviC9PT0Dt1RZ86cwefzAXDy5Emqq6tJTU3FbrcTGxvL4cOHMU2TsrIycnJyQtV0EZEBLShHHuvXr+fQoUOcPXuW5cuXM3/+fGbNmnVJlxXAoUOH2LZtGzExMVgsFpYsWeI/2f7www+zYcMGmpqayM7O1pVWIiIhEpTwWLVqVafzH3300UvmTZkyhSlTpnS6/OjRo6/o/IeIiPSt6DjDKiIiQaXwEBGRgCk8REQkYAoPEREJmMJDREQCpvAQEZGAKTxERCRgCg8REQmYwkNERAKm8BARkYApPEREJGAKDxERCZjCQ0REAqbwEBGRgCk8REQkYAoPEREJmMJDREQCFpSRBDds2EBlZSUJCQn+kQC3bdvGBx98wLBhwwBYsGABkyZNAmDnzp2UlpZisVhYvHgx2dnZAFRVVbF582Z8Ph+zZ89m7ty5wWi+iIhcJCjhMXPmTG6//XaKi4s7zJ8zZw533XVXh3knTpygvLycl156CY/Hw7p163j55ZcB2LRpE08//TRJSUmsXbuWnJwcrrnmmmCUICIi7QQlPMaNG4fL5erRshUVFUydOhWbzcbw4cNJS0vj6NGjAKSlpZGamgrA1KlTqaioUHiIiIRAUMKjK++//z5lZWVkZGTwwAMPEB8fj9vtJjMz07+Mw+HA7XYDkJSU5J+flJTEkSNHulx3SUkJJSUlAOTn55OcnNxPVVzK5/Nhtbqw2cBqtWK327FY+vb0ks/nwzBOYbPZ+m0boWK1WoO6v4IpWmuL1rogemvrbV0hC4/bbruNefPmAfDuu+/y9ttvk5eX12frdzqdOJ1O/+uampo+W3d3DMPA6/XS3NyM1wsejwfTNPt8G6Zp9us2QiU5OTmo+yuYorW2aK0Lore27upKT0+/7OdD9lM1MTERi8WCxWJh9uzZfPPNN0DrkUZtba1/ObfbjcPhuGR+bW0tDocj6O0WEZEQhofH4/FPf/7554wcORKAnJwcysvLaW5uxuVyUV1dzZgxYxg9ejTV1dW4XC68Xi/l5eXk5OSEqvkiIgNaULqt1q9fz6FDhzh79izLly9n/vz5HDx4kOPHj2MYBikpKSxduhSAkSNHctNNN/H4449jsVh46KGH/H35Dz74IM899xw+n49bb73VHzgiIhJcQQmPVatWXTJv1qxZXS6fm5tLbm7uJfMnTZrkvxdERERCJzouzxERkaBSeIiISMAUHiIiEjCFh4iIBEzhISIiAQvp40kkPBmG4Z+OlrvWRaRvKTykA8MweGv/KdwNzThibSyamKIAEZFLKDzkEu6GZmrONYe6GSISxnTOQ0REAqbwEBGRgCk8REQkYAoPEREJmMJDREQCpvAQEZGAKTxERCRgCg8REQlYUG4S3LBhA5WVlSQkJFBYWAjAli1b2LdvH1arldTUVPLy8rjqqqtwuVysXr3aP/h6Zmamf5TBY8eOUVxcTFNTExMnTmTx4sUdHqUhIiLBEZTwmDlzJrfffjvFxcX+eRMmTODee+8lJiaGd955h507d7Jw4UIA0tLSKCgouGQ9r7/+OsuWLSMzM5MXXniBqqoqJk6cGIwSRESknaB0W40bN474+PgO82644QZiYmIAyMrKwu12X3YdHo+HhoYGsrKyMAyD6dOnU1FR0W9tFhGRroXFs61KS0uZOnWq/7XL5eKpp54iNjaWn/3sZ4wdOxa3201SUpJ/maSkpMsGTklJCSUlJQDk5+eTnJzcfwVcxOfzYbW6sNnAarVit9uxWPo2p30+H4ZxCpvN1qfbCEbbu2O1WoO6v4IpWmuL1rogemvrbV0hD48dO3YQExPDtGnTALDb7WzYsIGhQ4dy7NgxCgoK/OdJAuF0OnE6nf7XNTU1fdbm7hiGgdfrpbm5Ga+39aipr59MaxgGpmn2+TaC0fbuJCcnB3V/BVO01hatdUH01tZdXW3nnbvS45+Un3zySafzP/30056u4hK7d+9m3759rFy50n/i22azMXToUAAyMjJITU2luroah8NBbW2t/7O1tbU4HI4r3raIiFy5HofHxo0bO53/2muvXdGGq6qqeO+99/jFL37B4MGD/fPPnDmDz+cD4OTJk1RXV5Oamordbic2NpbDhw9jmiZlZWXk5ORc0bZFRKR3uu22OnnyJNDaF+5yuTp0YZw8eZJBgwZ1u5H169dz6NAhzp49y/Lly5k/fz47d+7E6/Wybt064K+X5B46dIht27YRExODxWJhyZIl/pPtDz/8MBs2bKCpqYns7GxdaSUiEiLdhsfKlSv90ytWrOjwXmJiIv/4j//Y7UZWrVp1ybxZs2Z1uuyUKVOYMmVKp++NHj36is5/iIhI3+o2PN59910Ann32WX71q1/1e4MGGo0XLiKRqMdXWyk4+p7GCxeRSNXj8HC5XGzdupXjx4/T2NjY4b1/+7d/6/OGDRQaL1xEIlGPw+Pll18mNTWVBx54oMPVUSIiMvD0ODxOnDjBunXrgn638UBhoPMfIhI5ehweY8eO5fjx42RkZPRne6Je+4BoP50Ya+WtShe1Ov8hIhGgx+GRkpLCc889x09+8hMSExM7vHfPPff0ecOiRfsjCsMw2Fzpwt3Qeo5jlD2W9g+U1/kPEYkUPQ6PCxcu8Ld/+7e0tLR0eEyIXF77I4pR9lg87QLCHhvyR4uJiFyRHn975eXl9Wc7olrbEYXCQkSiRY+/zdoeU9KZ1NTUPmmMiIhEhh6HR/vHlFys7S506RvRfuVVNNcmMlD0ODwuDojTp0/zH//xH4wdO7bPGzXQRfOVV7qrXiQ6XPFNG4mJiSxatIh///d/78v2yF+0nSfxNDRjGEan/xGh37lttbVddSYikadXZ3C///57Lly40FdtkU5cfLXWmUavf1pEJFR6HB7PPPNMh77qCxcu8N133zFv3rx+aZj8VfurtU43eHXlloiEXI+/gS4ef2PIkCH8zd/8DVdffXWfN0pERMJbj8Nj5syZvdrQhg0bqKysJCEhwT+gU319PUVFRZw6dYqUlBRWr15NfHw8pmmyefNm9u/fz+DBg8nLy/M/FmX37t3s2LEDgNzc3F63S0REAtfjE+Zer5dt27bx2GOPcd999/HYY4+xbds2vF5vjz4/c+ZM/umf/qnDvF27djF+/HheeeUVxo8fz65duwDYv38/P/zwA6+88gpLly7ljTfeAFrDZvv27Tz//PM8//zzbN++nfr6+p6WICIifaTH4fHOO+/wxRdfsGTJEgoKCliyZAl//vOfeeedd3r0+XHjxvnHIm9TUVHBjBkzAJgxYwYVFRUA7N27l+nTp2MYBllZWZw7dw6Px0NVVRUTJkwgPj6e+Ph4JkyYQFVVVU9LEBGRPtLjbqtPP/2UgoIChg4dCkB6ejqjRo3iySefZNGiRVe08bq6Oux2O9B66W9dXR0Abreb5ORk/3JJSUm43W7cbjdJSUn++Q6HA7fb3em6S0pKKCkpASA/P7/D+vqbz+fDanVhs4E1xkpMDJdMQ9fv9XTaMLzYbDasVit2u71PHpffoe19uN5A1m+1WoO6v4IpWmuL1rogemvrbV09Do/+vpHLf+9CH3E6nTidTv/rmpqaPlt3dwzDwOv10tzcjLfFSkvLpdNAl+/1dNo0aZ32gsfj6ZN91KHtfbjeQNafnJwc1P0VTNFaW7TWBdFbW3d1paenX/bzPf5JedNNN/Hiiy9SVVXFiRMnqKqqoqCggClTpvS8tRdJSEjA4/EArV8iw4YNA1qPKNoXVVtbi8PhwOFwdHiir9vtxuFwXPH2RUTkyvQ4PBYuXMj48ePZtGkTa9as4c033+T666/n/vvvv+KN5+TksGfPHgD27NnD5MmT/fPLysowTZPDhw8TFxeH3W4nOzubAwcOUF9fT319PQcOHCA7O/uKtx8t2p6F1ddHbyIiXem22+qrr75i7969LFy4kHvuuafDwE/vvPMOx44dIysrq9sNrV+/nkOHDnH27FmWL1/O/PnzmTt3LkVFRZSWlvov1QWYOHEilZWVrFy5kkGDBvkfBx8fH8/dd9/N2rVrAZg3b94lJ+EHomh+FpaIhKduw2Pnzp38/d//fafvXX/99ezYsYM1a9Z0u6FVq1Z1Ov+ZZ565ZJ5hGDz88MOdLj9r1qxLblgUjUIoIsHVbbfV8ePHu+waGj9+PN9++22fN0pERMJbt+HR0NDQ5Y2ALS0tNDQ09HmjREQkvHUbHiNGjODAgQOdvnfgwAFGjBjR540SEZHw1m14zJkzh9/85jd89tln+Hw+oPVGr88++4zXX3+dOXPm9HsjI4WueBKRgaLbE+a33HILp0+fpri4mObmZoYNG8aZM2ew2WzMnz+fW265JRjtDHvtR8gbZY9F8SEi0axHd5jfeeedzJo1i8OHD1NfX098fDxZWVnExcX1d/siSvtxN0Il2sc/F5Hw0ONvubi4ON2QFwF0z4eIBIOGo4tCuudDRPpb3z0uVUREBgyFh4iIBEzdVr3UdnI6HC/P1clzEekvCo9eCPfLcy938lyhIiK9ofDopXC4PPdyOjt53j70dEWWiFyJ8PzGkz7XvgvLMAxdkSUivaLwGCDad2GFYxebiEQWXW01gLQdbZxu1BGHiPROSI88vv/+e4qKivyvXS4X8+fP59y5c3zwwQf+Mc0XLFjApEmTgNbBqUpLS7FYLCxevFh3vYuIhEBIwyM9PZ2CggKg9Um9y5Yt4yc/+Qkffvghc+bM4a677uqw/IkTJygvL+ell17C4/Gwbt06Xn75ZSwWHUCJiART2HzrfvHFF6SlpZGSktLlMhUVFUydOhWbzcbw4cNJS0vj6NGjQWyliIhAGJ0w//jjj7n55pv9r99//33KysrIyMjggQceID4+HrfbTWZmpn8Zh8OB2+3udH0lJSWUlJQAkJ+fT3Jycp+32efzYbW6sNnAGmMlJoYeT0Pgn7l42jC82Gy2Xq3HZrWSkJDgP3rz+XxYY/5Sk9WK3W7v0yO7Dv9mXazfarX2y/4KB9FaW7TWBdFbW2/rCovw8Hq97Nu3j3vvvReA2267jXnz5gHw7rvv8vbbb5OXlxfQOp1OJ06n0/+6pqam7xr8F4Zh4PV6aW5uxttipaWl59NAwJ+5eNo0ueLPtk3H26y8svsotQ2tbRplj/3rcl7weDx9eg9Ih3+zLtafnJzcL/srHERrbdFaF0Rvbd3VlZ6eftnPh0W31f79+xk1ahSJiYkAJCYmYrFYsFgszJ49m2+++QZoPdKora31f87tduNwOILe3mgbMbDtKixdiSUiPRUW4XFxl5XH4/FPf/7554wcORKAnJwcysvLaW5uxuVyUV1dzZgxY4La1ra7s18q/54dh9y6X0JEBqSQd1s1Njbypz/9iaVLl/rnvfPOOxw/fhzDMEhJSfG/N3LkSG666SYef/xxLBYLDz30UEiutAr3R5L0FT1YUUS6EvJvvyFDhvDmm292mLdixYoul8/NzSU3N7e/myV0vCs9KdbGoknD/QGiIBEZ2EIeHhLe2h9lKUhEpI3CQ3qssyDRU3lFBiaFh1yRtiDReRGRgUnhIb1yuQGnRCR6KTyk1zQ2iMjAo/CQPqMuLJGBQ+EhfUZdWCIDh8JD+lRXXVjth8AVkcin8JB+1/ZIF7eGwBWJGgoP6Rftz38YhjFgHukiMlDo/2TpF+3Pf+hoQyT6KDx6SH32gdPRhkj00v/VPaA+exGRjsJiPI9I0PYrWoMliYgoPERE5AooPEREJGBhcc7j0UcfZciQIVgsFmJiYsjPz6e+vp6ioiJOnTpFSkoKq1evJj4+HtM02bx5M/v372fw4MHk5eWRkZER6hJERAaUsAgPgGeffZZhw4b5X+/atYvx48czd+5cdu3axa5du1i4cCH79+/nhx9+4JVXXuHIkSO88cYbPP/88yFsuYjIwBO23VYVFRXMmDEDgBkzZlBRUQHA3r17mT59OoZhkJWVxblz5/B4PKFsqojIgBM2Rx7PPfccAH/3d3+H0+mkrq4Ou90OQGJiInV1dQC43W6Sk5P9n0tKSsLtdvuXbVNSUkJJSQkA+fn5HT4TKJ/Ph9XqwmYDa4yVmBh6PQ29X5dheLHZbGHVpoCmrVbsdjsWS8ffMFartVf7K5xFa23RWhdEb229rSsswmPdunU4HA7q6ur4l3/5F9LT0zu8bxhGwDfnOZ1OnE6n/3VNTc0Vt88wDLxeL83NzXhbrLS09H4a6PW6TJOwa1NA017weDyXPHk3OTm5V/srnEVrbdFaF0Rvbd3VdfH38MXCotvK4XAAkJCQwOTJkzl69CgJCQn+7iiPx+M/H+JwODoUXFtb6/98X2sLLd1VLiLSUcjDo7GxkYaGBv/0n/70J6699lpycnLYs2cPAHv27GHy5MkA5OTkUFZWhmmaHD58mLi4uEu6rPpC213lL5V/z45Dbt1VLiLSTsi7rerq6vjXf/1XAFpaWrjlllvIzs5m9OjRFBUVUVpa6r9UF2DixIlUVlaycuVKBg0aRF5eXr+1Tc9mEhHpXMi/FVNTUykoKLhk/tChQ3nmmWcumW8YBg8//HAwmib9TMPWikSukIeHDFwatlYkcik8JKS6GrZWRMKbwkPCQvsuLJ/PF9rGiEi3FB4SFtp3YQ0f6mHheHvAXVg6fyISPAoPCRttXVhWa+B/lu0H7NL5E5H+p/CQqKHzJyLBE/KbBEVEJPIoPEREJGAKDxERCZjOeUjYufjOc9DVUyLhRuEhYSdxyF8v2wV09ZRIGFJ4SFjSlVMi4U3hcZG27hKN4SEi0jWFRzvtbzQbZY/VGB4iIl3Q1VYXaesuOd2oLhMRka7oyEPCnsb9EAk/IQ2PmpoaiouLOX36NIZh4HQ6ueOOO9i2bRsffPCBf9zyBQsWMGnSJAB27txJaWkpFouFxYsXk52dHcoSJAg07odI+AlpeMTExHD//feTkZFBQ0MDa9asYcKECQDMmTOHu+66q8PyJ06coLy8nJdeegmPx8O6det4+eWXsVjU+xbtdPWVSHgJ6beu3W4nIyMDgNjYWEaMGIHb7e5y+YqKCqZOnYrNZmP48OGkpaVx9OjRYDVXwkBbF1bbfyISGmFzzsPlcvHtt98yZswYvvrqK95//33KysrIyMjggQceID4+HrfbTWZmpv8zDofjsmEj0UddWCLhISzCo7GxkcLCQhYtWkRcXBy33XYb8+bNA+Ddd9/l7bffJi8vL6B1lpSUUFJSAkB+fj7Jycndfsbn82G1urDZwBpjJSaGfpuG3q/LMLzYbLawalOftMMwiImxdtkmT4OXuiawWk3sdjsWi6XjvrNa/fPDjdVq7dHfYqSJ1rogemvrbV0hDw+v10thYSHTpk3jxhtvBCAxMdH//uzZs3nxxReB1iON2tpa/3tutxuHw9Hpep1OJ06n0/+6pqam27YYhoHX66W5uRlvi5WWlv6bBnq9LtMk7NrUJ+0wh/SsTV7weDyYptlx37WbH26Sk5N79LcYaaK1Loje2rqrKz09/bKfD+lPM9M02bhxIyNGjODOO+/0z/d4PP7pzz//nJEjRwKQk5NDeXk5zc3NuFwuqqurGTNmTNDbLSIy0IX0yOPrr7+mrKyMa6+9lieffBJovSz3448/5vjx4xiGQUpKCkuXLgVg5MiR3HTTTTz++ONYLBYeeuihsOyaEBGJdiENj+uuu45t27ZdMr/tno7O5Obmkpub25/NEhGRboT8nIfIlWp/57ku2xUJLoWHRKz2l+3qQZYiwaUTBhLR9CBLkdDQkYdEHT1IUaT/KTxQv3m00V3oIv1vwIeHBoCKTv3xIEUdzYj81YAPD/jrF409Vv8c0rn2PzJ0NCOi8JAod/H5j/YC/fLXY+FF/krhIVHt4st5zzR6qW1oJinWxqJJw/0BoqMIkcAoPCTqte+WPN3g9U93dVJd5zZEuqfwkAGrLVQuvlN9c6VL5zZEuqHwkAHv4q4tTxeh0qaz8ygKGBloFB4idH7FXVePP2k/H9ARigxICg+Ry+jqMm5deSUDncJDpJeu5HLgrk7K62S9RAqFh0gv9fRyYACfz4fFYun0pLxuRJRIovAQ6QPdXQ7cFip1zS5GDrV1eVK+J91hOjqRcBCR4VFVVcXmzZvx+XzMnj2buXPnhrpJIp26OFRON8FQ21/f7+qkfFddYVd6KbECR/paxIWHz+dj06ZNPP300yQlJbF27VpycnK45pprQt00kSvSkyu92rrCurqU+HJ6Gji9eap0S0tLp5+/XFD15LxPT9clwRdx4XH06FHS0tJITU0FYOrUqVRUVPQqPByxrT8FE4fYsGBgBmG6L7ZnizHxttjCqk19MZ0wxIqvxRZWbeqrf7MYq9nj5c80erv8ezWBH9ljee8rD3WNXkYMG0x9Uwt1f/lM+9cjhg3u8LToro5oOltXT6cbvGdwn2/qMD9hiJWfXmfvMqjattd+ua7acbl19TefzxeVwzX0tq6ICw+3201SUpL/dVJSEkeOHLlkuZKSEkpKSgDIz88nPT29y3Wuvfrqvm+oSIRZlpYWFtsLdjt64uoo/Y7oTV1ROwyt0+kkPz+f/Px81qxZE+rm9AvVFXmitbZorQuit7be1hVx4eFwOKitrfW/rq2txeFwhLBFIiIDT8SFx+jRo6mursblcuH1eikvLycnJyfUzRIRGVBifvnLX/4y1I0IhMViIS0tjVdffZX/+Z//Ydq0aUyZMqXbz2VkZAShdcGnuiJPtNYWrXVB9NbWm7oMU9e/iYhIgCKu20pEREJP4SEiIgGLuPs8AhEtjzGpqamhuLiY06dPYxgGTqeTO+64g5dHKCMAAApqSURBVPr6eoqKijh16hQpKSmsXr2a+Pj4UDf3ivh8PtasWYPD4WDNmjW4XC7Wr1/P2bNnycjIYMWKFVitkfXneu7cOTZu3Mh3332HYRg88sgjpKenR8U+++///m9KS0sxDIORI0eSl5fH6dOnI26fbdiwgcrKShISEigsLATo8v8r0zTZvHkz+/fvZ/DgweTl5YX1uZDOatuyZQv79u3DarWSmppKXl4eV111FQA7d+6ktLQUi8XC4sWLyc7OvvwGzCjV0tJiPvbYY+YPP/xgNjc3m0888YT53XffhbpZV8TtdpvffPONaZqmef78eXPlypXmd999Z27ZssXcuXOnaZqmuXPnTnPLli2hbGav/P73vzfXr19vvvDCC6ZpmmZhYaH5xz/+0TRN03zttdfM999/P5TNuyKvvvqqWVJSYpqmaTY3N5v19fVRsc9qa2vNvLw888KFC6Zptu6rDz/8MCL32cGDB81vvvnGfPzxx/3zutpH+/btM5977jnT5/OZX3/9tbl27dqQtLmnOqutqqrK9Hq9pmm21tlW23fffWc+8cQTZlNTk3ny5EnzscceM1taWi67/qjttmr/GBOr1ep/jEkkstvt/l84sbGxjBgxArfbTUVFBTNmzABgxowZEVtfbW0tlZWVzJ49G2h9htHBgwf9V9HNnDkz4mo7f/48X375JbNmzQLAarVy1VVXRc0+8/l8NDU10dLSQlNTE4mJiRG5z8aNG3fJkV9X+2jv3r1Mnz4dwzDIysri3LlzeDyeoLe5pzqr7YYbbiAmJgaArKws3G430Frz1KlTsdlsDB8+nLS0NI4ePXrZ9Yf3MWUv9PQxJpHG5XLx7bffMmbMGOrq6rDb7QAkJiZSV1cX4tZdmbfeeouFCxfS0NAAwNmzZ4mLi/P/kTscDv8feaRwuVwMGzaMDRs28H//939kZGSwaNGiqNhnDoeDf/iHf+CRRx5h0KBB3HDDDWRkZET8PmvT1T5yu90kJyf7l0tKSsLtdvuXjTSlpaVMnToVaK0tMzPT/15P9l/UHnlEo8bGRgoLC1m0aBFxcXEd3jMMIyIf3rZv3z4SEhLCuu/4SrS0tPDtt99y22238etf/5rBgweza9euDstE6j6rr6+noqKC4uJiXnvtNRobG6mqqgp1s/pFpO6j7uzYsYOYmBimTZt2xeuI2iOPaHuMidfrpbCwkGnTpnHjjTcCkJCQgMfjwW634/F4GDZsWIhbGbivv/6avXv3sn//fpqammhoaOCtt97i/PnztLS0EBMTg9vtjrh9l5SURFJSkv/X3JQpU9i1a1dU7LMvvviC4cOH+9t+44038vXXX0f8PmvT1T5yOBzU1NT4l4vU75Tdu3ezb98+nnnmGX8wXvx92ZP9F7VHHtH0GBPTNNm4cSMjRozgzjvv9M/Pyclhz549AOzZs4fJkyeHqolX7N5772Xjxo0UFxezatUqrr/+elauXMmPf/xjPv30U6D1jz3S9l1iYiJJSUl8//33QOsX7jXXXBMV+yw5OZkjR45w4cIFTNP01xbp+6xNV/soJyeHsrIyTNPk8OHDxMXFRVyXVVVVFe+99x6/+MUvGDx4sH9+Tk4O5eXlNDc343K5qK6uZsyYMZddV1TfYV5ZWclvf/tbfD4ft956K7m5uaFu0hX56quveOaZZ7j22mv9vxQWLFhAZmYmRUVF1NTURPRln20OHjzI73//e9asWcPJkydZv3499fX1jBo1ihUrVmCz2bpfSRg5fvw4GzduxOv1Mnz4cPLy8jBNMyr22bZt2ygvLycmJoYf/ehHLF++HLfbHXH7bP369Rw6dIizZ8+SkJDA/PnzmTx5cqf7yDRNNm3axIEDBxg0aBB5eXmMHj061CV0qbPadu7cidfr9f/NZWZmsnTpUqC1K+vDDz/EYrGwaNEiJk6ceNn1R3V4iIhI/4jabisREek/Cg8REQmYwkNERAKm8BARkYApPEREJGAKD5FuHDx4kOXLl4e6GSJhJWrvMBfpzP333++fbmpqwmq1YrG0/oZaunRprx7XEIl++ctfMm3aNP9DKUV6SuEhA8qWLVv8048++ijLli1jwoQJIWyRSGRSeIgAzc3N/O53v+OTTz4B4KabbuK+++7r9A7pP/zhD/zv//4vTz/9NMOGDWPr1q188skneL1eJk+ezKJFixg0aBAHDx7k1VdfZc6cObz33ntYLBYWLFjArbfe2mkb6uvrefvttzlw4ABNTU2MHTuWp556CoCSkhLee+896uvrue6661iyZAkOhwOXy8Vjjz3G1q1b/U+0bX80sXv3bj744AMyMzP58MMPiYuL4+GHH2bixIls3bqVL7/8kiNHjvDWW28xc+ZMHnrooX76F5Zoo3MeIrQ+muHIkSP8+te/pqCggKNHj/Kf//mflyy3fft29uzZw69+9SuSkpL43e9+R3V1NQUFBbzyyiu43W62b9/uX/706dOcP3+ejRs3snz5cjZt2kR9fX2nbXj11Ve5cOEChYWFvP766/7nmP35z39m69atrF69mt/85jekpKTw8ssv97i2o0ePkp6ezqZNm/jpT3/Kxo0bMU2TBQsWMHbsWB588EG2bNmi4JCAKDxEgD/+8Y/cfffdJCQkMGzYMObNm8dHH33kf980TX77299y4MABnn32WYYNG4ZpmnzwwQf8/Oc/Jz4+ntjYWHJzc/n444/9n4uJiWHevHlYrVYmTZrEkCFD/A9LbM/j8VBVVcWSJUuIj4/HarUybtw4AD766CNuvfVWMjIysNls3HvvvRw+fBiXy9Wj2pKTk3E6nVgsFmbMmIHH44nIcUQkvKjbSoTWR1CnpKT4X6ekpHQYDOf8+fOUlJSwevVq/1gqZ86c4cKFC6xZs8a/nGma+Hw+/+uhQ4f6u5MABg8eTGNj4yXbr62tJT4+vtOHJHo8HkaNGuV/PWTIEOLj43v82PPExMQO2wc6bYNIIBQeIrSOZ3Dq1ClGjhwJQE1NTYcv5quuuooVK1ZQVFTEE088wXXXXcfQoUMZNGgQL730Uq/HdUhKSqK+vp5z585x1VVXdXjPbrd3GEeisbGR+vp6HA4HQ4YMAeDChQv+UDt9+nSPtxuNAx1JcKjbSgS4+eab2bFjB2fOnOHMmTNs3779kst2f/zjH7Ny5UoKCws5evQoFouF2bNn89Zbb3UYqvRKRtWz2+1kZ2fzxhtvUF9fj9fr5dChQ/62ffjhhxw/fpzm5ma2bt3KmDFj/AMyORwOPvroI3w+H6WlpZw8ebLH201ISAhoeZE2Cg8RIDc3l4yMDJ544gmeeOIJRo0a1en4LxMmTOCRRx7hxRdf5NixY9x3332kpaXxz//8z/z85z9n3bp1nZ7T6IkVK1YQExPD6tWrWbJkCX/4wx/827znnnsoLCxk6dKlnDx5klWrVvk/t2zZMv7rv/6LBx98kBMnTpCVldXjbd5xxx189tlnLF68mDfffPOK2i0Dk8bzEBGRgOnIQ0REAqbwEBGRgCk8REQkYAoPEREJmMJDREQCpvAQEZGAKTxERCRgCg8REQnY/wNrWGyykmndtAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(len_tokens)\n",
    "plt.xlim([0, 123])\n",
    "plt.xlabel('Token count')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qNFUMrkNeGK7"
   },
   "source": [
    "The histogram shows that most of the tweets have under 60 tokens and maximum of 123. We will use the maximun length of 125 for our models."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Q0Wt5c5E1QYl"
   },
   "source": [
    "<h2> 6. Built customised dataset constructor, classifier methods and training function </h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "G8rZeqcL4d4W"
   },
   "source": [
    "<h3> 6.1 Building the data class to feed to the model </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "id": "VlXF-k0J4eWK"
   },
   "outputs": [],
   "source": [
    "class EmotionsDataset(Dataset):\n",
    "\n",
    "    def __init__(self, content, targets, tokenizer, max_length):\n",
    "        self.content = content\n",
    "        self.targets = targets\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_length = max_length\n",
    "  \n",
    "    def __getitem__(self, idx):\n",
    "        tweet = str(self.content[idx])\n",
    "        target = self.targets[idx]\n",
    "\n",
    "        encoded_sent = self.tokenizer.encode_plus(tweet,\n",
    "          add_special_tokens=True,\n",
    "          return_token_type_ids=False,\n",
    "          max_length = self.max_length,\n",
    "          padding = 'max_length',\n",
    "          truncation = True,\n",
    "          return_attention_mask=True,\n",
    "          return_tensors='pt')\n",
    "\n",
    "        o = dict()\n",
    "        o['attention_mask'] = encoded_sent['attention_mask'].flatten()\n",
    "        o['input_ids'] = encoded_sent['input_ids'].flatten()\n",
    "        o['sent_text'] = tweet\n",
    "        o['targets']= torch.tensor(target, dtype=torch.long)\n",
    "        return o\n",
    "   \n",
    "    def __len__(self):\n",
    "\n",
    "        return len(self.content)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vqtP5ztas6W-"
   },
   "source": [
    "<h3> 6.2 Building BERT, DistilBert and RoBERTa Multiclass Sequence Classifiers </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "id": "YVy1utpVth4_"
   },
   "outputs": [],
   "source": [
    "class EmotionClassifiers(nn.Module):\n",
    "\n",
    "    def __init__(self, base_model, hidden=None, num_classes=2, p=0.2, freeze_params=False):\n",
    "\n",
    "        super(EmotionClassifiers, self).__init__()\n",
    "\n",
    "        if hidden is None:\n",
    "            hidden = [512, 512]\n",
    "        self.base_model = base_model \n",
    "        self.num_classes = num_classes\n",
    "        self.hidden = hidden\n",
    "        self.dropout = nn.Dropout(p)\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(base_model.config.hidden_size, self.hidden[0]),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.2),\n",
    "            nn.Linear(self.hidden[0], self.hidden[1]),\n",
    "            nn.Dropout(0.2),\n",
    "            nn.Linear(self.hidden[1], self.num_classes))\n",
    "\n",
    "        if freeze_params:\n",
    "            for param in self.base_model.parameters():\n",
    "                param.requires_grad = False\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "\n",
    "        output = self.base_model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        sequence_output = output[0]\n",
    "        pooled_output = sequence_output[:, 0]\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        logits = self.classifier(pooled_output)\n",
    "\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eNUJOraXaFHZ"
   },
   "source": [
    "<h3> 6.3 Training one epoch function </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "id": "S03lC_hlWOra"
   },
   "outputs": [],
   "source": [
    "def train_one_epoch(model, data_loader, loss_fn, optimizer, scheduler):\n",
    "    \"\"\"\n",
    "    :param model: model to train\n",
    "    :param data_loader: constructor to pass batches of data to process\n",
    "    :param loss_fn: loss/ cost function to minimise\n",
    "    :param optimizer: optimizer to minimise the loss\n",
    "    :param scheduler: policy to reduce the learning rate\n",
    "    :return: running loss and accuracy\n",
    "    \"\"\"\n",
    "\n",
    "    model = model.to(device)\n",
    "    model.train()\n",
    "\n",
    "    epoch_loss, correct = 0, 0\n",
    "\n",
    "    for batch in data_loader:\n",
    "\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['targets'].to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        output = model(input_ids, attention_mask)\n",
    "        loss = loss_fn(output, labels)\n",
    "        epoch_loss += loss.item()\n",
    "        \n",
    "        preds = torch.argmax(output, dim=-1)\n",
    "        correct += (preds == labels).sum().item()\n",
    "\n",
    "        loss.backward()\n",
    "        # nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) \n",
    "        # prevents exploding gradients\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "\n",
    "    return epoch_loss/len(data_loader),  correct/len(data_loader.dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "W2vqxGtmaPWL"
   },
   "source": [
    "<h3> 6.4 Evalute function </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "id": "9NcaoSg71s2A"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, data_loader):\n",
    "    \"\"\"\n",
    "    :param model: model to evaluate\n",
    "    :param data_loader: constructor to process the data\n",
    "    :return: running loss and accuracy \n",
    "    \"\"\"\n",
    "\n",
    "    model.eval()\n",
    "\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "    eval_loss, correct = 0, 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "\n",
    "        for batch in (data_loader):\n",
    "\n",
    "            input_ids = batch['input_ids'].to(device)\n",
    "            attention_mask = batch['attention_mask'].to(device)\n",
    "            labels = batch['targets'].to(device)\n",
    "            # get logits\n",
    "            output = model(input_ids, attention_mask) \n",
    "            # add loss\n",
    "            eval_loss += loss_fn(output, labels).item()\n",
    "\n",
    "            # compute predictions\n",
    "            preds = torch.argmax(output, dim=-1) \n",
    "            correct += (preds == labels).sum().item()\n",
    "\n",
    "    return eval_loss/len(data_loader), correct/len(data_loader.dataset)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-K9WNCEGrCPr"
   },
   "source": [
    "<h3> 6.5 Building the main training function </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "id": "3RxnWx_XA3RA"
   },
   "outputs": [],
   "source": [
    "def main(**kwargs):\n",
    "    \"\"\"\n",
    "    :param kwargs: a dictionary to create the model , loader and parametert to train \n",
    "    and validate the model.\n",
    "    :return: a dictionary with training stats, scheduler and the model.\n",
    "    \"\"\"\n",
    "\n",
    "    train_dataset = EmotionsDataset(kwargs['x_train'], kwargs['y_train'], \n",
    "                                    kwargs['tokenizer'], max_length=125)\n",
    "    val_dataset = EmotionsDataset(kwargs['x_val'], kwargs['y_val'], \n",
    "                                  kwargs['tokenizer'], max_length=125)\n",
    "\n",
    "    model =  EmotionClassifiers(kwargs['base_model'], kwargs['hidden'],\n",
    "                                kwargs['num_classes'], kwargs['dropout'])\n",
    "\n",
    "    train_dataloader = DataLoader(train_dataset, batch_size=kwargs['batch'],\n",
    "                                  shuffle=True, num_workers=2)\n",
    "    val_dataloader = DataLoader(val_dataset, batch_size=kwargs['batch'], \n",
    "                                shuffle= True, num_workers=2)\n",
    "\n",
    "    no_decay = ['bias', 'LayerNorm.weight']\n",
    "    grouped_parameters = [{'params': [p for n, p in model.named_parameters()\n",
    "                                          if not any(nd in n for nd in no_decay)],\n",
    "                               'weight_decay': kwargs['decay']},\n",
    "                              {'params': [p for n, p in model.named_parameters()\n",
    "                                          if any(nd in n for nd in no_decay)], \n",
    "                               'weight_decay': 0.0}]\n",
    "\n",
    "    optimizer = torch.optim.AdamW(grouped_parameters, lr=kwargs['lr'])\n",
    "    total_steps = len(train_dataloader) * kwargs['epochs']\n",
    "\n",
    "    if kwargs['scheduler'] == 'linear':\n",
    "        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0,\n",
    "                                             num_training_steps=total_steps)\n",
    "    elif kwargs['scheduler'] == 'one_cycle':\n",
    "        scheduler = OneCycleLR(optimizer, max_lr= kwargs['lr'], \n",
    "                               steps_per_epoch=len(train_dataloader), \n",
    "                               epochs=kwargs['epochs'], pct_start=0.2)\n",
    "    else:\n",
    "        assert kwargs['scheduler'] in ['linear', 'one_cycle'], \\\n",
    "            'Input linear or one_cycle scheduler'\n",
    "\n",
    "    best_acc = -np.inf\n",
    "    stats_log = defaultdict(list)\n",
    "    n_steps = 0\n",
    "\n",
    "    for epoch in tqdm(range(kwargs['epochs'])):\n",
    "\n",
    "        epoch_start_time = time.time()\n",
    "\n",
    "        train_loss, train_acc = train_one_epoch(model, train_dataloader,\n",
    "                                                      kwargs['loss'], optimizer, scheduler)\n",
    "\n",
    "        print(scheduler.get_last_lr())\n",
    "        n_steps += len(train_dataloader)\n",
    "        val_loss, val_acc = evaluate(model, val_dataloader)\n",
    "\n",
    "        print('| End of epoch {:2d} | time: {:.2f}s | train loss {:.4f} | '\n",
    "            'train_acc {:.2f}%'.format(epoch+1, (time.time() - epoch_start_time),\n",
    "                                        train_loss, train_acc*100))\n",
    "\n",
    "        print('| Steps {}| val loss {:.4f} | '\n",
    "            'val_acc {:.2f}%'.format(n_steps, val_loss, val_acc*100))\n",
    "     \n",
    "\n",
    "        # save the best model\n",
    "        if val_acc > best_acc:\n",
    "            best_acc = val_acc\n",
    "            torch.save({\n",
    "              'epoch': epoch + 1,\n",
    "              'model': model.state_dict(),\n",
    "              'optimizer': optimizer.state_dict(),'scheduler':scheduler.state_dict(),\n",
    "              'acc': best_acc, 'lr': scheduler.get_last_lr(),\n",
    "              'stats': stats_log\n",
    "              }, os.path.join(kwargs['output_dir'], 'best_model_{}.pt'.format(kwargs['exp'])))\n",
    "            print('saved')\n",
    "\n",
    "        stats_log['train_acc'].append(train_acc)\n",
    "        stats_log['train_loss'].append(train_loss)\n",
    "        stats_log['val_acc'].append(val_acc)\n",
    "        stats_log['val_loss'].append(val_loss)\n",
    "        stats_log['steps'].append(n_steps)\n",
    "\n",
    "    return stats_log, model, scheduler\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Om5rmsMVFuhb"
   },
   "source": [
    "<h2>7. Experimental setup</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "w2GjVwWkI4pW"
   },
   "source": [
    "A total 24 experiments will be run as product of the combination of the following:\n",
    " \n",
    " - Base models - BERT, DistilBERT, RoBERTa\n",
    " - The loss function to be minimised - wCE and CE (Cross entropy and Weighted Cross entropy)\n",
    " - Schedulers - linear and one cycle learning rate policies\n",
    " - Learning rate - two rates at which the algorithm learns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "id": "QGnjkoXGFriA"
   },
   "outputs": [],
   "source": [
    "exps =[i+1 for i in range(0, 24)]\n",
    "models = ['bert', 'distilbert', 'roberta'] * 8\n",
    "exp_loss= ['wCE' if i > 11 else 'CE' for i in range(0, 24)]\n",
    "lr_rates = [8e-6 if i > 5 else 1e-5 for i in range(0, 12)] * 2\n",
    "schedulers =  ['linear' if i > 2 else 'one_cycle' for i in range(0, 6)] * 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "id": "moNDshVyIL0t"
   },
   "outputs": [],
   "source": [
    "setup = pd.DataFrame({'Exp':exps,'Network': models, 'Loss': exp_loss, 'lr':lr_rates, \n",
    "                      'scheduler': schedulers})\n",
    "setup.set_index('Exp', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GfhzpzlndbCk"
   },
   "source": [

    "\n",
    "| Experiment |  Network   | Loss | Learning<br />rate | Learning<br/>policy |\n",
    "|:----------:|:----------:|:----:|:------------------:|:-------------------:|\n",
    "| 01         |    BERT    |  CE  |       1e-05        |     one\\_cycle      |\n",
    "| 02         | DistilBERT |  CE  |       1e-05        |     one\\_cycle      |\n",
    "| 03         |  RoBERTa   |  CE  |       1e-05        |     one\\_cycle      |\n",
    "| 04         |    BERT    |  CE  |       1e-05        |       linear        |\n",
    "| 05         | DistilBERT |  CE  |       1e-05        |       linear        |\n",
    "| 06         |  RoBERTa   |  CE  |       1e-05        |       linear        |\n",
    "| 07         |    BERT    |  CE  |       8e-06        |     one\\_cycle      |\n",
    "| 08         | DistilBERT |  CE  |       8e-06        |     one\\_cycle      |\n",
    "| 09         |  RoBERTa   |  CE  |       8e-06        |     one\\_cycle      |\n",
    "| 10         |    BERT    |  CE  |       8e-06        |       linear        |\n",
    "| 11         | DistilBERT |  CE  |       8e-06        |       linear        |\n",
    "| 12         |  RoBERTa   |  CE  |       8e-06        |       linear        |\n",
    "| 13         |    BERT    | wCE  |       1e-05        |     one\\_cycle      |\n",
    "| 14         | DistilBERT | wCE  |       1e-05        |     one\\_cycle      |\n",
    "| 15         |  RoBERTa   | wCE  |       1e-05        |     one\\_cycle      |\n",
    "| 16         |    BERT    | wCE  |       1e-05        |       linear        |\n",
    "| 17         | DistilBERT | wCE  |       1e-05        |       linear        |\n",
    "| 18         |  RoBERTa   | wCE  |       1e-05        |       linear        |\n",
    "| 19         |    BERT    | wCE  |       8e-06        |     one\\_cycle      |\n",
    "| 20         | DistilBERT | wCE  |       8e-06        |     one\\_cycle      |\n",
    "| 21         |  RoBERTa   | wCE  |       8e-06        |     one\\_cycle      |\n",
    "| 22         |    BERT    | wCE  |       8e-06        |       linear        |\n",
    "| 23         | DistilBERT | wCE  |       8e-06        |       linear        |\n",
    "| 24         |  RoBERTa   | wCE  |       8e-06        |       linear        | \n",
    "    \n"

   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IRuTm8vwSx7X"
   },
   "source": [
    "<h3> 7.1 Loss function </h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hpNvshIuIL5-"
   },
   "source": [
    "- **Building weights for cross entropy loss fuction -Imbalaced dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "id": "U6l8VtPQLNJt"
   },
   "outputs": [],
   "source": [
    "class_weights = class_weight.compute_class_weight(class_weight='balanced', \n",
    "                                                classes=np.unique(y_train), y=y_train)\n",
    "weights = torch.tensor(class_weights, dtype=torch.float)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4I7L2j__oCBr"
   },
   "source": [
    "- __Building the loss function__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "id": "xDMaGkDnwDR8"
   },
   "outputs": [],
   "source": [
    "# https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html \n",
    "\n",
    "# Weights for imbalanced dataset - feed the weights to the cross entropy\n",
    "w_criterion = nn.CrossEntropyLoss(weight=weights).to(device)\n",
    "\n",
    "# Cross entropy\n",
    "criterion = nn.CrossEntropyLoss().to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YcBIc_ssY6Dp"
   },
   "source": [
    "<h3> 7.2 Preparing tokenizers and base models for training </h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EL-HQpPQCJcf"
   },
   "source": [
    "\n",
    "- **Prepare tokenizers**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "id": "MLpnePpHxg_1"
   },
   "outputs": [],
   "source": [
    "# Create the tokenizers\n",
    "bert_tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased', \n",
    "                                                   clean_text=True)\n",
    "dist_tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased', \n",
    "                                                         clean_text=True)\n",
    "rob_tokenizer = RobertaTokenizerFast.from_pretrained('roberta-base', \n",
    "                                                     clean_text=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "G1SfnRX-Cd5S"
   },
   "source": [
    "- **Extract the base models to buil the classifiers**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "v_wmGBpsxP_l",
    "outputId": "91dbc085-0226-49ae-c9f1-abfd2335a76c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_transform.weight']\n",
      "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaModel: ['lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.bias', 'lm_head.dense.weight', 'lm_head.decoder.weight', 'lm_head.dense.bias']\n",
      "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "# Get the base model to build the classifiers\n",
    "bert = BertModel.from_pretrained('bert-base-uncased')\n",
    "distbert = DistilBertModel.from_pretrained('distilbert-base-uncased')\n",
    "roberta = RobertaModel.from_pretrained('roberta-base')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ut14eRsgEuB6"
   },
   "source": [
    "<h2> 8. Training the classifier </h2>\n",
    "\n",
    "<h3> 8.1 Training hyparameters </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "id": "hgCwNypE9ttU"
   },
   "outputs": [],
   "source": [
    "# Create an output diretory to store the weights\n",
    "outdir = os.path.join(os.getcwd(), 'weights')\n",
    "os.makedirs(outdir, exist_ok=True)\n",
    "\n",
    "exp = 24\n",
    "exp_model = setup.Network.loc[exp]\n",
    "exp_loss = setup.Loss.loc[exp]\n",
    "lr = setup.lr.loc[exp]\n",
    "policy = setup.scheduler.loc[exp]\n",
    "\n",
    "if exp_loss == 'CE':\n",
    "    loss = criterion\n",
    "else:\n",
    "    loss = w_criterion\n",
    "\n",
    "if exp_model == 'bert': \n",
    "    tokenizer = bert_tokenizer\n",
    "    base_model = bert\n",
    "\n",
    "if exp_model == 'distilbert': \n",
    "    tokenizer = dist_tokenizer\n",
    "    base_model = distbert\n",
    "\n",
    "else:\n",
    "    tokenizer = rob_tokenizer\n",
    "    base_model = roberta\n",
    "\n",
    "# Dictionary of hyperameters to run the experiments\n",
    "params = {'exp': exp, 'base_model': base_model, 'tokenizer': tokenizer, \n",
    "          'hidden': [768, 512], 'num_classes': len(emotions),'dropout': 0.3, 'batch': 16,\n",
    "          'lr': lr, 'decay': 0.01, 'x_train': X_train, 'y_train': y_train, 'x_val': X_val,\n",
    "          'y_val':y_val, 'loss': loss, 'epochs': 4, 'scheduler': policy, \n",
    "          'output_dir': outdir}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yxrOgW9ykzcD"
   },
   "source": [
    "<h3> 8.2 Start training</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZgbWYtJPnaFJ"
   },
   "outputs": [],
   "source": [
    "print('Exp: {} | Model: {} | Loss: {} | Learning rate: {} | '\n",
    "      'Policy: {}'.format(exp, exp_model, exp_loss, lr, policy))\n",
    "\n",
    "# Invoking method to start training\n",
    "train_stats, model_, last_lr = main(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OQPzBNwEiiDr"
   },
   "outputs": [],
   "source": [
    "print('Last learning rate: ', last_lr.get_last_lr())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bf4xsdKl1QYo"
   },
   "source": [
    "<h3> 8.3 Extracting performance metrics from training  and validation</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "id": "_G9lma9CA3RE"
   },
   "outputs": [],
   "source": [
    "tr_acc, tr_loss, v_acc, v_loss= train_metrics(**train_stats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O8ih_EbmTOMF"
   },
   "source": [
    "<h3> 8.4 Visualising training results </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "id": "7M-u3okcF-I6"
   },
   "outputs": [],
   "source": [
    "def plot_stats(exp=exp, hideplot=False, n_epochs=4, **stats):\n",
    "    \"\"\"\n",
    "    :param hideplot: boolean to show or training and validation plot\n",
    "    :param stats: a dictionary with the loss and accuracy for training and validation\n",
    "    :return: plots for accuracy and  loss\n",
    "    \"\"\"\n",
    "    \n",
    "    plots_dir = os.path.join(os.getcwd(), 'plots')\n",
    "    os.makedirs(plots_dir, exist_ok=True)\n",
    "    \n",
    "    # Plot the loss per epoch\n",
    "    fig1 = plt.figure(figsize=(8,4))\n",
    "    x =  np.arange(1,  n_epochs+1) \n",
    "    plt.plot(x, stats['train_loss'], label='train')\n",
    "    plt.plot(x, stats['val_loss'], label='val')\n",
    "    plt.xticks(range(1, n_epochs+1))\n",
    "    plt.title(\"Loss trend\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.ylabel(\"Loss\")\n",
    "    plt.legend()\n",
    "    plt.savefig(os.path.join(plots_dir, 'loss_{}.png'.format(exp)))\n",
    "    if hideplot:\n",
    "        plt.close(fig1)\n",
    "    else:\n",
    "        plt.rcParams.update({'font.size': 10})\n",
    "        plt.show(fig1)\n",
    "\n",
    "    # Plot accuracy per epoch\n",
    "    fig2 = plt.figure(figsize=(8,4))\n",
    "    x =  np.arange(1,  n_epochs+1) \n",
    "    plt.plot(x, stats['train_acc'], label='train')\n",
    "    plt.plot(x, stats['val_acc'], label='val')\n",
    "    plt.xticks(range(1, n_epochs+1))\n",
    "    plt.title(\"Accuracy trend\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.ylabel(\"Accuracy\")\n",
    "    plt.legend()\n",
    "    plt.savefig(os.path.join(plots_dir, 'acc_{}.png'.format(exp)))\n",
    "    if hideplot:\n",
    "        plt.close(fig2)\n",
    "    else:\n",
    "        plt.rcParams.update({'font.size': 10})\n",
    "        plt.show(fig2)\n",
    "\n",
    "    return fig1, fig2\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zcPR90OwLhyG"
   },
   "outputs": [],
   "source": [
    "plot_stats(exp, hideplot=False, n_epochs=params['epochs'], **train_stats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XcF-fvAQMW_A"
   },
   "source": [
    "<h2> 9. Evaluate the model on the test dataset\n",
    "\n",
    "### **Metrics**\n",
    "\n",
    "- Accuracy \n",
    "- Balanced Accuracy\n",
    "- F1 Score\n",
    "- Matthew's Correlation Coefficient\n",
    "- Precision\n",
    "- Recall\n",
    "- AUROC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QHHawIr91QYp"
   },
   "source": [
    "<h3> 9.1 Build test dataset and dataloader </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "id": "a8r_uKxRH3BV"
   },
   "outputs": [],
   "source": [
    "test_dataset = EmotionsDataset(X_test, y_test, params['tokenizer'], max_length=125)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "id": "g-70RDCS5NAH"
   },
   "outputs": [],
   "source": [
    "prediction_sampler = SequentialSampler(test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "id": "KrclEst9A3RB"
   },
   "outputs": [],
   "source": [
    "test_dataloader = DataLoader(test_dataset, batch_size= 16, sampler=prediction_sampler)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GzPLO4Im1QYq"
   },
   "source": [
    "<h3>9.2 Load best trained weights onto model</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "id": "5N1Nq3SexF1Y"
   },
   "outputs": [],
   "source": [
    "checkpoint=torch.load(os.path.join(outdir, 'best_model_{}.pt'.format(params['exp'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Avwl__esxvNl"
   },
   "outputs": [],
   "source": [
    "model_.load_state_dict(checkpoint['model'], strict=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SsTqfCgB1QYq"
   },
   "source": [
    "<h3> 9.3 Get predictions </h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iguYl9UC1QYq"
   },
   "source": [
    "-  **Build function to read predictions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "id": "P11UBrBgTgBg"
   },
   "outputs": [],
   "source": [
    "def get_predictions(model, data_loader):\n",
    "    \"\"\"\n",
    "    Calculates predictions on test dataset\n",
    "    returns list of probabilities, true and predicted labels\n",
    "    \"\"\"\n",
    "\n",
    "    model = model.eval()\n",
    "\n",
    "    sent_texts = []\n",
    "    predictions = []\n",
    "    prediction_probs = []\n",
    "    true_labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for batch in data_loader:\n",
    "\n",
    "            texts = batch[\"sent_text\"]\n",
    "            input_ids = batch[\"input_ids\"].to(device)\n",
    "            attention_mask = batch[\"attention_mask\"].to(device)\n",
    "            targets = batch[\"targets\"].to(device)\n",
    "\n",
    "            output = model(\n",
    "            input_ids=input_ids,\n",
    "            attention_mask=attention_mask\n",
    "            )\n",
    "            preds = torch.argmax(output, dim=1)\n",
    "            probs = F.softmax(output, dim=1)\n",
    "\n",
    "            sent_texts.extend(texts)\n",
    "            predictions.extend(preds)\n",
    "            prediction_probs.append(probs)\n",
    "            true_labels.extend(targets)\n",
    "\n",
    "    predictions = torch.stack(predictions).cpu()\n",
    "    prediction_probs = torch.cat(prediction_probs)\n",
    "    true_values = torch.stack(true_labels).cpu()\n",
    "\n",
    "    return sent_texts, predictions, prediction_probs, true_values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3AuJybbv1QYq"
   },
   "source": [
    "- **Compute predictions and probabilities**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "id": "xKUho8RJTrIn"
   },
   "outputs": [],
   "source": [
    "y_review_texts, y_pred, y_probs, y_true = get_predictions(\n",
    "  model_, test_dataloader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tnlJv7cz1QYr"
   },
   "source": [
    "- **Save true labels**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "id": "0hYp1FQN1QYr"
   },
   "outputs": [],
   "source": [
    "if exp == 1:\n",
    "    true_dir = os.path.join(os.getcwd(), 'true_labels')\n",
    "    # create a directory for the true labels\n",
    "    os.makedirs(true_dir, exist_ok=True)\n",
    "    # save the true labels in npz format\n",
    "    true_labels = save_file(y_true, true_dir, 'true_labels{}.npz'.format(exp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "id": "8MowCc0WaMKk"
   },
   "outputs": [],
   "source": [
    "# create a directory for the prediction\n",
    "pred_path = os.path.join(os.getcwd(), 'predictions')\n",
    "os.makedirs(pred_path, exist_ok=True)\n",
    "\n",
    "# Save predictions in npz format\n",
    "save_file(y_pred, pred_path, 'pred_exp{}.npz'.format(exp))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MX4R7ijrA3RD"
   },
   "source": [
    "<h3> 9.4 Evaluation Metrics </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "id": "u5pPzs_bA3RD"
   },
   "outputs": [],
   "source": [
    "metrics = Metrics(y_true, y_pred, y_probs, emotions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lTE11SXBA3RD"
   },
   "outputs": [],
   "source": [
    "# Show confusion matrix\n",
    "metrics.cmatrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1gs8l0znA3RD"
   },
   "outputs": [],
   "source": [
    "print('Classification report', metrics.classification_rep(), sep='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "spZ-0wgxbn0q"
   },
   "outputs": [],
   "source": [
    "ba = metrics.balanced_acc()\n",
    "mcc = metrics.mcc()\n",
    "acc = metrics.acc()\n",
    "auroc = metrics.auroc()\n",
    "precision, recall, f1, pre_class, rec_class, f1_class = metrics.macro_average()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "id": "wsOqUSIyB-Wd"
   },
   "outputs": [],
   "source": [
    "epoch= checkpoint['epoch'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "id": "I6fvdQZaUlP7"
   },
   "outputs": [],
   "source": [
    "metrics_ = [['Date', date.today()], ['exp', params['exp']],\n",
    "            ['Architecture', exp_model], ['loss', exp_loss], \n",
    "            ['lr', params['lr']], ['Test acc', np.round(acc,4)], \n",
    "            ['scheduler', params['scheduler']], ['BA', np.round(ba, 4)], \n",
    "            ['F1 macro', np.round(f1, 4)],\n",
    "            ['Recall', np.round(recall, 4)],\n",
    "            ['Precision', np.round(precision, 4)],\n",
    "            ['MCC', np.round(mcc, 4)],\n",
    "            ['AUROC', np.round(auroc.item(),4)],\n",
    "            ['F1 per class', np.round(f1_class, 4)],\n",
    "            ['Recall per class', np.round(rec_class, 4)], \n",
    "             ['Pres per class', np.round(pre_class, 4)], ['epoch', epoch],\n",
    "            ['train_acc', np.round(tr_acc, 4)], ['val_acc', np.round(v_acc, 4)]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "id": "ENAl_ToqWCgy"
   },
   "outputs": [],
   "source": [
    "df_metrics = pd.DataFrame(metrics_, columns= ['Metric','Value'],\n",
    "                          index = [i for i in range(len(metrics_))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "id": "Mkhx5JCwWp4P"
   },
   "outputs": [],
   "source": [
    "# Transpose  the evaluation results and saved them into csv file\n",
    "df_metrics= df_metrics.T\n",
    "df_metrics.rename(columns=df_metrics.iloc[0], inplace=True)\n",
    "df_metrics.drop(df_metrics.index[0], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "id": "QMy7krzxWFN-"
   },
   "outputs": [],
   "source": [
    "# Save metrics into csv file\n",
    "fname = 'exp_' + str(exp) + '.csv'\n",
    "stats_dir = os.path.join(os.getcwd(), 'metrics')\n",
    "os.makedirs(stats_dir, exist_ok=True)\n",
    "df_metrics.to_csv(os.path.join(stats_dir, fname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NxyyKkMJ1QYs"
   },
   "source": [
    "<h3> 10. Ranking the models performance </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "id": "lQ1xIjoY7B9s"
   },
   "outputs": [],
   "source": [
    "eval_dir= 'metrics'\n",
    "models_list = natsorted(os.listdir('metrics'))\n",
    "models_list =  [m for m in models_list if m != '.DS_Store']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "id": "0HeIy8BJ7B9s"
   },
   "outputs": [],
   "source": [
    "df2_metrics = get_metrics(models_list, eval_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "id": "jK1vb9lT7B9t"
   },
   "outputs": [],
   "source": [
    "df2_metrics = df2_metrics[['Test acc', 'BA', 'F1 macro', 'Recall', 'Precision', 'MCC', 'epoch']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "id": "FqyYkLNE7B9t"
   },
   "outputs": [],
   "source": [
    "df2_metrics.rename(columns={'Test acc': 'Acc', 'F1 macro': 'F1', 'Recall': 'Rec', \n",
    "                             'Precision': 'Prec', 'epoch': 'Epoch'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "id": "B0owoBaekMmd"
   },
   "outputs": [],
   "source": [
    "ranks = df2_metrics[['Acc', 'BA', 'F1', 'Rec', 'Prec', 'MCC']].rank(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "id": "4e9wNIxo7B9t"
   },
   "outputs": [],
   "source": [
    "ranks['Epoch'] = df2_metrics['Epoch'].rank( ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "id": "iSnnW2psdsth"
   },
   "outputs": [],
   "source": [
    "setup_rank = pd.concat([setup, ranks], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "id": "npcAGRN77B9t"
   },
   "outputs": [],
   "source": [
    "ranked_metrics = pd.concat([df2_metrics, ranks], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "id": "X2W40wbw9H7i"
   },
   "outputs": [],
   "source": [
    "# create a directory for the ranks\n",
    "ranks_csv = os.path.join(os.getcwd(), 'ranks')\n",
    "os.makedirs(ranks_csv, exist_ok=True)\n",
    "\n",
    "# Save ranks in csv file\n",
    "fname = 'all_ranks' + '.csv'\n",
    "ranked_metrics.to_csv(os.path.join(ranks_csv, fname))\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "emotion_analysis2.ipynb",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}