07_introduction_to_nlp_in_tensorflow.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Introduction to NLP Fundamentals in TensorFlow\n",
        "\n",
        "NLP has the goal of deriving information out of natural language (could be sequences text or speech).\n",
        "\n",
        "Another common term for NLP problems is sequence to sequence problems (seq2seq)."
      ],
      "metadata": {
        "id": "lGTGXfUQukQu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Install tensorflow version 2.15\n",
        "!pip install tensorflow==2.15"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AjVO-LIHsa1a",
        "outputId": "400a6993-08bc-4f0f-da50-449a9bd11d09"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tensorflow==2.15\n",
            "  Downloading tensorflow-2.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.4 kB)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=23.5.26 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (24.3.25)\n",
            "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (0.6.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (0.2.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (3.11.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (18.1.1)\n",
            "Collecting ml-dtypes~=0.2.0 (from tensorflow==2.15)\n",
            "  Downloading ml_dtypes-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB)\n",
            "Requirement already satisfied: numpy<2.0.0,>=1.23.5 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (1.26.4)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (3.3.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (24.1)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (3.20.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (71.0.4)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (1.16.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (2.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (4.12.2)\n",
            "Collecting wrapt<1.15,>=1.11.0 (from tensorflow==2.15)\n",
            "  Downloading wrapt-1.14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (0.37.1)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.15) (1.64.1)\n",
            "Collecting tensorboard<2.16,>=2.15 (from tensorflow==2.15)\n",
            "  Downloading tensorboard-2.15.2-py3-none-any.whl.metadata (1.7 kB)\n",
            "Collecting tensorflow-estimator<2.16,>=2.15.0 (from tensorflow==2.15)\n",
            "  Downloading tensorflow_estimator-2.15.0-py2.py3-none-any.whl.metadata (1.3 kB)\n",
            "Collecting keras<2.16,>=2.15.0 (from tensorflow==2.15)\n",
            "  Downloading keras-2.15.0-py3-none-any.whl.metadata (2.4 kB)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow==2.15) (0.44.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow==2.15) (2.27.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow==2.15) (1.2.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow==2.15) (3.7)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow==2.15) (2.32.3)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow==2.15) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow==2.15) (3.0.3)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow==2.15) (5.5.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow==2.15) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow==2.15) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow==2.15) (1.3.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.16,>=2.15->tensorflow==2.15) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.16,>=2.15->tensorflow==2.15) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.16,>=2.15->tensorflow==2.15) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.16,>=2.15->tensorflow==2.15) (2024.7.4)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.16,>=2.15->tensorflow==2.15) (2.1.5)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow==2.15) (0.6.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow==2.15) (3.2.2)\n",
            "Downloading tensorflow-2.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (475.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m475.2/475.2 MB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading keras-2.15.0-py3-none-any.whl (1.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m53.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading ml_dtypes-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m50.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tensorboard-2.15.2-py3-none-any.whl (5.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.5/5.5 MB\u001b[0m \u001b[31m79.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tensorflow_estimator-2.15.0-py2.py3-none-any.whl (441 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m442.0/442.0 kB\u001b[0m \u001b[31m32.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading wrapt-1.14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (77 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: wrapt, tensorflow-estimator, ml-dtypes, keras, tensorboard, tensorflow\n",
            "  Attempting uninstall: wrapt\n",
            "    Found existing installation: wrapt 1.16.0\n",
            "    Uninstalling wrapt-1.16.0:\n",
            "      Successfully uninstalled wrapt-1.16.0\n",
            "  Attempting uninstall: ml-dtypes\n",
            "    Found existing installation: ml-dtypes 0.4.0\n",
            "    Uninstalling ml-dtypes-0.4.0:\n",
            "      Successfully uninstalled ml-dtypes-0.4.0\n",
            "  Attempting uninstall: keras\n",
            "    Found existing installation: keras 3.4.1\n",
            "    Uninstalling keras-3.4.1:\n",
            "      Successfully uninstalled keras-3.4.1\n",
            "  Attempting uninstall: tensorboard\n",
            "    Found existing installation: tensorboard 2.17.0\n",
            "    Uninstalling tensorboard-2.17.0:\n",
            "      Successfully uninstalled tensorboard-2.17.0\n",
            "  Attempting uninstall: tensorflow\n",
            "    Found existing installation: tensorflow 2.17.0\n",
            "    Uninstalling tensorflow-2.17.0:\n",
            "      Successfully uninstalled tensorflow-2.17.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorstore 0.1.64 requires ml-dtypes>=0.3.1, but you have ml-dtypes 0.2.0 which is incompatible.\n",
            "tf-keras 2.17.0 requires tensorflow<2.18,>=2.17, but you have tensorflow 2.15.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed keras-2.15.0 ml-dtypes-0.2.0 tensorboard-2.15.2 tensorflow-2.15.0 tensorflow-estimator-2.15.0 wrapt-1.14.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check TensorFlow version\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NRWxmhXAs6It",
        "outputId": "158efc0f-b874-4b7d-9f99-e9a9c81eee82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.15.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Get helper functions"
      ],
      "metadata": {
        "id": "ZDJ5vvoJvbER"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/helper_functions.py\n",
        "\n",
        "# Import series of helper functions for the notebook\n",
        "from helper_functions import unzip_data, create_tensorboard_callback, plot_loss_curves, compare_historys"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ItULornLvtRL",
        "outputId": "ecee7577-278e-4159-96e2-6e6f11bbe68b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-08-26 16:01:47--  https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/helper_functions.py\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 10246 (10K) [text/plain]\n",
            "Saving to: ‘helper_functions.py’\n",
            "\n",
            "helper_functions.py 100%[===================>]  10.01K  --.-KB/s    in 0s      \n",
            "\n",
            "2024-08-26 16:01:48 (116 MB/s) - ‘helper_functions.py’ saved [10246/10246]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Get a text dataset\n",
        "\n",
        "The dataset we're going to be using is Kaggle's introduction to NLP dataset (text samples of Tweets labelled as disaster or not disaster)."
      ],
      "metadata": {
        "id": "eS1xb8I6v62z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget \"https://storage.googleapis.com/ztm_tf_course/nlp_getting_started.zip\"\n",
        "\n",
        "# Unzip data\n",
        "unzip_data(\"nlp_getting_started.zip\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KIYddzMmwWUf",
        "outputId": "51ed42ba-262e-4e82-efd1-22aeb05673bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-08-26 16:16:49--  https://storage.googleapis.com/ztm_tf_course/nlp_getting_started.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 142.251.2.207, 142.250.141.207, 74.125.137.207, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|142.251.2.207|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 607343 (593K) [application/zip]\n",
            "Saving to: ‘nlp_getting_started.zip.1’\n",
            "\n",
            "\r          nlp_getti   0%[                    ]       0  --.-KB/s               \rnlp_getting_started 100%[===================>] 593.11K  --.-KB/s    in 0.004s  \n",
            "\n",
            "2024-08-26 16:16:49 (143 MB/s) - ‘nlp_getting_started.zip.1’ saved [607343/607343]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Visualizing a text dataset\n",
        "\n",
        "To visualize our text samples, we first need to read them in, one way to do so would be to use Python: https://realpython.com/read-write-files-python/\n",
        "\n",
        "But we prefer to get visual straight away. So another way to use this is to use **Pandas.**"
      ],
      "metadata": {
        "id": "N1rnUCf7wqeJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "train_df = pd.read_csv(\"train.csv\")\n",
        "test_df = pd.read_csv(\"test.csv\")\n",
        "train_df.head()"
      ],
      "metadata": {
        "id": "PpBWE1fQd-_2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "d19545ca-347d-4220-8cfc-519f32faa9a9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   id keyword location                                               text  \\\n",
              "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
              "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
              "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
              "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
              "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
              "\n",
              "   target  \n",
              "0       1  \n",
              "1       1  \n",
              "2       1  \n",
              "3       1  \n",
              "4       1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-44532721-2af8-4f46-9426-cf72f56aca21\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>All residents asked to 'shelter in place' are ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-44532721-2af8-4f46-9426-cf72f56aca21')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-44532721-2af8-4f46-9426-cf72f56aca21 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-44532721-2af8-4f46-9426-cf72f56aca21');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-086ad8e9-d7ea-4086-a1f8-0fb76be31b58\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-086ad8e9-d7ea-4086-a1f8-0fb76be31b58')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-086ad8e9-d7ea-4086-a1f8-0fb76be31b58 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "train_df",
              "summary": "{\n  \"name\": \"train_df\",\n  \"rows\": 7613,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3137,\n        \"min\": 1,\n        \"max\": 10873,\n        \"num_unique_values\": 7613,\n        \"samples\": [\n          3796,\n          3185,\n          7769\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"keyword\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 221,\n        \"samples\": [\n          \"injury\",\n          \"nuclear%20reactor\",\n          \"engulfed\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"location\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3341,\n        \"samples\": [\n          \"Oklahoma\",\n          \"Starling City\",\n          \"Trinidad and Tobago\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 7503,\n        \"samples\": [\n          \"Three Homes Demolished in Unrecognized Arab Village - International Middle East Media Center http://t.co/ik8m4Yi9T4\",\n          \"Reid Lake fire prompts campground evacuation order http://t.co/jBODKM6rBU\",\n          \"FAAN orders evacuation of abandoned aircraft at MMA http://t.co/dEvYbnVXGQ via @todayng\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Shuffle training dataframe\n",
        "train_df_shuffled = train_df.sample(frac=1, random_state=69)\n",
        "train_df_shuffled.head()"
      ],
      "metadata": {
        "id": "OGimnbVTfpU_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "ae21aba0-00cf-4376-ad92-d3c88ab001f3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        id             keyword            location  \\\n",
              "421    611            arsonist  [ Blonde Bi Fry. ]   \n",
              "5047  7195  natural%20disaster               home    \n",
              "5046  7194            mudslide                 NaN   \n",
              "1881  2702               crush                 NaN   \n",
              "6543  9362            survived                 NaN   \n",
              "\n",
              "                                                   text  target  \n",
              "421                         'wHeRE's mY aRsOnISt aT???'       0  \n",
              "5047  she's a natural disaster she's the last of the...       0  \n",
              "5046  @nikistitz even the one that looked like a mud...       0  \n",
              "1881                        kenny holland crush da vida       0  \n",
              "6543  Not one character in the final destination ser...       0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e7068605-b8f1-426f-8f3f-055dd3372479\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>421</th>\n",
              "      <td>611</td>\n",
              "      <td>arsonist</td>\n",
              "      <td>[ Blonde Bi Fry. ]</td>\n",
              "      <td>'wHeRE's mY aRsOnISt aT???'</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5047</th>\n",
              "      <td>7195</td>\n",
              "      <td>natural%20disaster</td>\n",
              "      <td>home</td>\n",
              "      <td>she's a natural disaster she's the last of the...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5046</th>\n",
              "      <td>7194</td>\n",
              "      <td>mudslide</td>\n",
              "      <td>NaN</td>\n",
              "      <td>@nikistitz even the one that looked like a mud...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1881</th>\n",
              "      <td>2702</td>\n",
              "      <td>crush</td>\n",
              "      <td>NaN</td>\n",
              "      <td>kenny holland crush da vida</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6543</th>\n",
              "      <td>9362</td>\n",
              "      <td>survived</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Not one character in the final destination ser...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e7068605-b8f1-426f-8f3f-055dd3372479')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e7068605-b8f1-426f-8f3f-055dd3372479 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e7068605-b8f1-426f-8f3f-055dd3372479');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-493472ff-d623-47e5-8f07-7139b805e482\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-493472ff-d623-47e5-8f07-7139b805e482')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-493472ff-d623-47e5-8f07-7139b805e482 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "train_df_shuffled",
              "summary": "{\n  \"name\": \"train_df_shuffled\",\n  \"rows\": 7613,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3137,\n        \"min\": 1,\n        \"max\": 10873,\n        \"num_unique_values\": 7613,\n        \"samples\": [\n          4027,\n          6064,\n          8406\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"keyword\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 221,\n        \"samples\": [\n          \"hostage\",\n          \"bridge%20collapse\",\n          \"fire\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"location\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3341,\n        \"samples\": [\n          \"Dublin\",\n          \"Kensington, MD\",\n          \"Lindenhurst\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 7503,\n        \"samples\": [\n          \"So when r u getting married'@Foxy__Siren: Oh finally Jennifer Aniston got married??????... I'm so happy for her ??????'\",\n          \"im feeling attacked http://t.co/91jvYCxXVi\",\n          \"I-90 Vantage: Backups eastbound with low visibility due to a dust storm about 4 miles east of the bridge.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# What does the test dataframe look like ?\n",
        "test_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "NF09TzplgRLH",
        "outputId": "f2cd8686-c905-4c11-97a8-e4aa25fb39ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   id keyword location                                               text\n",
              "0   0     NaN      NaN                 Just happened a terrible car crash\n",
              "1   2     NaN      NaN  Heard about #earthquake is different cities, s...\n",
              "2   3     NaN      NaN  there is a forest fire at spot pond, geese are...\n",
              "3   9     NaN      NaN           Apocalypse lighting. #Spokane #wildfires\n",
              "4  11     NaN      NaN      Typhoon Soudelor kills 28 in China and Taiwan"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-61e36319-2311-4a1a-986f-7e51fbd3700f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Just happened a terrible car crash</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Heard about #earthquake is different cities, s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>there is a forest fire at spot pond, geese are...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>9</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>11</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-61e36319-2311-4a1a-986f-7e51fbd3700f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-61e36319-2311-4a1a-986f-7e51fbd3700f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-61e36319-2311-4a1a-986f-7e51fbd3700f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-5ac8de5d-fa3b-41e7-b35f-0dfa584dc99c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5ac8de5d-fa3b-41e7-b35f-0dfa584dc99c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-5ac8de5d-fa3b-41e7-b35f-0dfa584dc99c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "test_df",
              "summary": "{\n  \"name\": \"test_df\",\n  \"rows\": 3263,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3146,\n        \"min\": 0,\n        \"max\": 10875,\n        \"num_unique_values\": 3263,\n        \"samples\": [\n          8051,\n          425,\n          1330\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"keyword\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 221,\n        \"samples\": [\n          \"injury\",\n          \"nuclear%20reactor\",\n          \"engulfed\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"location\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1602,\n        \"samples\": [\n          \"UAE\",\n          \"Tokio / Tokyo\",\n          \"Texas\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3243,\n        \"samples\": [\n          \"Latest: USA: Huge sinkhole swallows up Brooklyn intersection http://t.co/vspKHg3nZy\",\n          \"I liked a @YouTube video http://t.co/a5YTAw9Vih S.O.S. Rona Guide - The Red Whirlwind\",\n          \"HitchBot travels Europe and greeted with open arms. Gets destroyed after two weeks in america. There's a lesson to be learned here.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 141
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# How many examples of each classes are there ?\n",
        "train_df.target.value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 178
        },
        "id": "sckyDte2gahe",
        "outputId": "fd75e233-a06d-4125-bc0c-374616f374ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "target\n",
              "0    4342\n",
              "1    3271\n",
              "Name: count, dtype: int64"
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>target</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4342</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3271</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div><br><label><b>dtype:</b> int64</label>"
            ]
          },
          "metadata": {},
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# How many total samples are there ?\n",
        "len(train_df), len(test_df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KNYXwU9cgoK0",
        "outputId": "2dde67a9-aecd-40b0-fbef-e58bf9a16137"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(7613, 3263)"
            ]
          },
          "metadata": {},
          "execution_count": 143
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's visualize some random training examples\n",
        "import random\n",
        "random_index = random.randint(0, len(train_df)-5) #create random indexes not higher than the total number of samples\n",
        "for row in train_df_shuffled[[\"text\", \"target\"]][random_index:random_index+5].itertuples():\n",
        "  _, text, target = row\n",
        "  print(f\"Target: {target}\", \"(real disaster)\" if target > 0 else \"(not real disaster)\")\n",
        "  print(f\"Text:\\n{text}\\n\")\n",
        "  print(\"---\\n\")"
      ],
      "metadata": {
        "id": "eS9bbwqUg_1H",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "921d30b0-35f1-46dc-c87a-3a80d6e18d4f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target: 1 (real disaster)\n",
            "Text:\n",
            "Rly tragedy in MP: Some live to recount horror: ÛÏWhen I saw coaches of my train plunging into water I called ... http://t.co/0Xh758OnUP\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0 (not real disaster)\n",
            "Text:\n",
            "I didn`t want to hurt you but the fear drove me to do it ............... Midnight\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0 (not real disaster)\n",
            "Text:\n",
            "Set goals &amp; DEMOLISH them all! ?\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0 (not real disaster)\n",
            "Text:\n",
            "If you dotish to blight your car go right ahead. Once it's not mine.\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1 (real disaster)\n",
            "Text:\n",
            "@tinybaby @adultblackmale @mattytalks meltdown mwednesday\n",
            "\n",
            "---\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Split data into training and validation sets"
      ],
      "metadata": {
        "id": "v7WacSs9iGin"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "id": "5z1qM3pni3EY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Use train_test_split to split training data into training and validation sets\n",
        "train_sentences, val_sentences, train_labels, val_labels = train_test_split(train_df_shuffled[\"text\"].to_numpy(),\n",
        "                                                                            train_df_shuffled[\"target\"].to_numpy(),\n",
        "                                                                            test_size=0.1,# use 10% of training data for validation\n",
        "                                                                            random_state=69)"
      ],
      "metadata": {
        "id": "RslbaIHCi8hF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the lengths\n",
        "len(train_sentences), len(train_labels), len(val_sentences), len(val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CeyZAcclj85e",
        "outputId": "ee9431e0-9546-4f24-f22a-b38231e8553d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(6851, 6851, 762, 762)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the first 10 samples\n",
        "train_sentences[:10], train_labels[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nGUZ_eerkG6t",
        "outputId": "3d44ea5a-8481-48e1-82e4-4ee386bec4b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array(['@ChrisDanielShow Nearly-Fatal Pee in San Francisco http://t.co/1tvlFrhm1m',\n",
              "        \"babe I'm gonna ruin you if you let me stay\",\n",
              "        \"We're happily collided :)\",\n",
              "        \"RT THR 'RT THRArchives: 1928: When Leo the MGM Lion Survived a Plane Crash #TBT http://t.co/Wpkl2qNiQW http://t.co/BD52FxDvhQ'\",\n",
              "        '@FoxNews let me report it to u people instead Mr.Obama just declares CNMI federal disaster area post typhoon soudelor. U guys 2slow2report.',\n",
              "        \"Don't say @ALIPAPER: women got problems this #keepingtheviginaclean thing is very interesting n less expensive #metrofmtalk'\",\n",
              "        \"'Jet fuel cant melt steel'\\n'The structural failure is illogical'\\n'The second plane crashing into the building is fake'\\n'It was a bomb'\",\n",
              "        'A grade in Black Horse Famine[MEGA]. Score 0840728\\nhttp://t.co/pdiit0AF3Q\\n#Dynamix http://t.co/ZQ5KDOx7BY',\n",
              "        'Even when I was a kid haha super late but folks used to bash me for that shit I understand he survived cancer but he still cheated',\n",
              "        '@montetjwitter11 @nolesfan05 @NutsAndBoltsSP I say concern but not panic. Too many games left. And some games left against each other.'],\n",
              "       dtype=object),\n",
              " array([0, 0, 0, 1, 1, 0, 1, 0, 0, 0]))"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Converting text into numbers\n",
        "\n",
        "When dealing with text problems , one of the first things you'll need to do before building a nmodel is to convert text into numbers.\n",
        "\n",
        "There are a few ways to do this, namely:\n",
        "* Tokenization - direct mapping of token (a token could be a word or a character) to number\n",
        "* Embedding - create a matrix of feature vector for each token (the size of the feature vector can be defined and this embedding can be learned)"
      ],
      "metadata": {
        "id": "mbYYRE0dkdvd"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Text vectorization (tokenization)"
      ],
      "metadata": {
        "id": "tvTphahNla4B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import TextVectorization\n",
        "\n",
        "# Use the default TextVectorization parameters\n",
        "text_vectorizer = TextVectorization(max_tokens=10000, # how many words in the vocabulary (all of the different words in your text & automatically add <OOV>)\n",
        "                                    standardize=\"lower_and_strip_punctuation\", # how to process text\n",
        "                                    split=\"whitespace\", # how to split tokens\n",
        "                                    ngrams=None, # create groups of n-words?\n",
        "                                    output_mode=\"int\", # how to map tokens to numbers\n",
        "                                    output_sequence_length=None,\n",
        "                                    pad_to_max_tokens=True) # how long should the output sequence of tokens be?"
      ],
      "metadata": {
        "id": "RJugJg53pMAx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the average number of tokens (words) in the training tweets\n",
        "round(sum([len(i.split()) for i in train_sentences])/len(train_sentences))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Br5ZxrysOBM",
        "outputId": "eae5c94b-2f66-4f05-b397-6bbf0423eeb0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "15"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Setup text vectorization variables\n",
        "max_vocab_length = 10000 # max number of words to have in our vocabulary\n",
        "max_length = 15 # max length our sequences will be (e.g. how many words will our model see from a Tweet?)\n",
        "\n",
        "text_vectorizer = TextVectorization(max_tokens=max_vocab_length,\n",
        "                                    output_mode=\"int\",\n",
        "                                    output_sequence_length=max_length)"
      ],
      "metadata": {
        "id": "FcPV6NWNuYHZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the text vectorizer to our training text\n",
        "text_vectorizer.adapt(train_sentences)"
      ],
      "metadata": {
        "id": "BP7BlF0tvmoo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a sample sentence and tokenize it\n",
        "sample_sentence = \"There's a flood in my street\"\n",
        "text_vectorizer([sample_sentence])\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-Q-FzHZyzsuG",
        "outputId": "50ab4f96-99d1-4256-d711-838b2f022a5e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\n",
              "array([[257,   3, 239,   5,  13, 702,   0,   0,   0,   0,   0,   0,   0,\n",
              "          0,   0]])>"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Choose a random sentence from our training dataset\n",
        "random_sentence = random.choice(train_sentences)\n",
        "print(f\"Original text:\\n{random_sentence}\\\n",
        "      \\n\\nVectorized version:\")\n",
        "text_vectorizer([random_sentence])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OvZ62luY0B9O",
        "outputId": "f187fb4b-4c2d-4197-a3cb-e7aa26a3a1e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original text:\n",
            "Stupid women nearly collided into me today after she came out of a junction not looking. Still kept coming towards me till I beep my horn      \n",
            "\n",
            "Vectorized version:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\n",
              "array([[1609,  457,  839,  303,   66,   26,  136,   41,  196,  421,   36,\n",
              "           6,    3,    1,   33]])>"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the unique words in the vocabulary\n",
        "words_in_vocab = text_vectorizer.get_vocabulary() # get all of the unique words in our training data\n",
        "top_5_words = words_in_vocab[:5] # get the most common words\n",
        "bottom_5_words = words_in_vocab[-5:] # get the least common words\n",
        "print(f\"Number of words in vocab: {len(words_in_vocab)}\")\n",
        "print(f\"Top 5 most common words: {top_5_words}\")\n",
        "print(f\"Bottom 5 least common words: {bottom_5_words}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kIsB5L4n0hwW",
        "outputId": "1bd6dbc6-f3d5-4cd8-ab43-814ac3d65e89"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of words in vocab: 10000\n",
            "Top 5 most common words: ['', '[UNK]', 'the', 'a', 'to']\n",
            "Bottom 5 least common words: ['panicked', 'pandora', 'pandemoniumiso', 'pandemic', 'panda']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Creating an Embedding using an Embedding layer\n",
        "\n",
        "To make our embedding, we're gonna use TensorFlow's embedding layer: https://www.tensorflow.org/api_docs/python/tf/keras/layers/Embedding\n",
        "\n",
        "The parameters we care most about for our embedding layer:\n",
        "* `input_dim` = the size of our vocabulary\n",
        "* `output_dim` = the size of the output embedding vector, for example, a value of 100 would mean each token gets represented by a vector 100 long\n",
        "* `input_length` = length of the sequences being passed to the embedding layer"
      ],
      "metadata": {
        "id": "Y2NUmT4z1lHl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras import layers\n",
        "\n",
        "embedding = layers.Embedding(input_dim=max_vocab_length, #set input shape\n",
        "                             output_dim=128, #output shape\n",
        "                             embeddings_initializer=\"uniform\",\n",
        "                             input_length=max_length #how long is each input\n",
        "                             )\n",
        "embedding"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T-1893Pi3fFp",
        "outputId": "45286b81-bf61-4b19-b178-5d7e335d67e7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.layers.core.embedding.Embedding at 0x7aa2bf542620>"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a random sentence from the training set\n",
        "random_sentence = random.choice(train_sentences)\n",
        "print(f\"Original text:\\n{random_sentence}\\\n",
        "      \\n\\nEmbedded version:\")\n",
        "\n",
        "# Embed the random sentence (turn it into dense vectors of fixed size)\n",
        "sample_embed = embedding(text_vectorizer([random_sentence]))\n",
        "sample_embed"
      ],
      "metadata": {
        "id": "eZfrrOmR4IiV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0481d97b-85b9-4f43-e43e-788bc90c0047"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original text:\n",
            "Everything has sunk in except the fact that I am actually moving to the state of Colorado tomorrow. Been dreaming of it since I was a kid.??      \n",
            "\n",
            "Embedded version:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15, 128), dtype=float32, numpy=\n",
              "array([[[ 0.00636565,  0.00592681,  0.00358897, ..., -0.01084081,\n",
              "         -0.02213096, -0.00346457],\n",
              "        [-0.04003441, -0.03282468,  0.04214465, ..., -0.0074444 ,\n",
              "          0.04960043, -0.02172639],\n",
              "        [ 0.04068836,  0.02085599, -0.04270485, ..., -0.02796459,\n",
              "          0.02744306,  0.03984911],\n",
              "        ...,\n",
              "        [-0.02230558,  0.04222805, -0.03502367, ...,  0.02244448,\n",
              "          0.04641071,  0.01419706],\n",
              "        [ 0.0094371 ,  0.02161688,  0.02060472, ...,  0.04133889,\n",
              "          0.01444495,  0.01420912],\n",
              "        [-0.00747392,  0.0199254 , -0.01110135, ...,  0.03875181,\n",
              "         -0.04933609,  0.0426794 ]]], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check out a single token embedding\n",
        "sample_embed[0][0], sample_embed[0][0].shape, random_sentence"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gADaP4Wm4f8D",
        "outputId": "51bdd6a7-0a16-45c1-8317-a7e920dc6ca7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
              " array([ 0.00636565,  0.00592681,  0.00358897,  0.00534964,  0.00436587,\n",
              "        -0.01361718,  0.02591332, -0.03729271, -0.00698181,  0.00970882,\n",
              "        -0.01243133, -0.01642624,  0.03250423, -0.02135881, -0.00098584,\n",
              "         0.0456293 ,  0.04400161, -0.01775271,  0.03560206,  0.03987877,\n",
              "        -0.04508553, -0.01209326, -0.04704343,  0.04747901, -0.01533603,\n",
              "         0.00015562, -0.0015602 ,  0.02665054,  0.04085553,  0.0445067 ,\n",
              "         0.04689126, -0.03652652,  0.03012563, -0.04621495, -0.02144338,\n",
              "         0.02042068,  0.00521366,  0.03142276,  0.00728102, -0.00927367,\n",
              "        -0.01318737,  0.02145623,  0.04055803, -0.00419555,  0.04442736,\n",
              "         0.01002332, -0.03560114,  0.04917708,  0.03242168, -0.04976051,\n",
              "         0.04561937, -0.01038402,  0.04137191,  0.02373941, -0.01848875,\n",
              "         0.019524  , -0.01055177,  0.00153286, -0.04953199,  0.02372998,\n",
              "         0.04326013,  0.04313648, -0.00027329,  0.01762129,  0.03331436,\n",
              "        -0.03083354, -0.04088135, -0.03479702,  0.03329423, -0.00127543,\n",
              "         0.02217751, -0.03970378,  0.0282698 , -0.00111643, -0.04616767,\n",
              "        -0.04200494, -0.01985481,  0.0019474 , -0.01677021, -0.04553661,\n",
              "         0.02755965, -0.04513751,  0.03938102,  0.00955937,  0.02964753,\n",
              "        -0.02999573,  0.02914726, -0.02395198, -0.00692546, -0.03137087,\n",
              "         0.02276417, -0.01624004,  0.01270868, -0.03032756, -0.02917452,\n",
              "        -0.04479644, -0.01188401, -0.01115177, -0.00569976,  0.01064912,\n",
              "        -0.02275456,  0.04066057, -0.03141589, -0.02772173, -0.00965965,\n",
              "        -0.03521911,  0.04247382, -0.03253881,  0.02811496, -0.04168107,\n",
              "        -0.02628036, -0.03521181,  0.04301915, -0.04654241, -0.04633754,\n",
              "        -0.00342257, -0.03494539, -0.0365649 , -0.01626972, -0.00759799,\n",
              "         0.03610018,  0.0135732 ,  0.00257417, -0.00865023, -0.01529734,\n",
              "        -0.01084081, -0.02213096, -0.00346457], dtype=float32)>,\n",
              " TensorShape([128]),\n",
              " 'Everything has sunk in except the fact that I am actually moving to the state of Colorado tomorrow. Been dreaming of it since I was a kid.??')"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Modelling a text dataset (runninga aseries of experiments)\n",
        "\n",
        "Now we've got a way to turn our text data into numbers, it's time to start building a series of modelling experiments.\n",
        "\n",
        "We'll start with a baseline and move on from there.\n",
        "\n",
        "* Model 0: Naive Bayes (baseline), this is from Sklearn ML map: https://scikit-learn.org/stable/tutorial/machine_learning_map/index.html\n",
        "* Model 1: Feed-forward neural network (dense model)\n",
        "* Model 2: LSTM model (RNN)\n",
        "* Model 3: GRU model (RNN)\n",
        "* Model 4: Bidirectional-LSTM model (RNN)\n",
        "* Model 5: 1D Convolutional Neural Network (CNN)\n",
        "* Model 6: TensorFlow Hub Pretrained Feature Extractor (using transfer learning from NLP)\n",
        "* Model 7: Same as model 6 with 10% of training data\n",
        "\n",
        "How are we going to approach all these ?\n",
        "\n",
        "Use the standard steps in modelling with tensorflow:\n",
        "* Create a model\n",
        "* Build a model\n",
        "* Fit a model\n",
        "* Evaluate our model"
      ],
      "metadata": {
        "id": "68yZJ3Ii47NN"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model 0: Getting a baseline\n",
        "\n",
        "As with all machine learning modelling experiments, it's important to create a baseline model so you've got a benchmark for future experiments to build upon.\n",
        "\n",
        "For the baseline, we'll use Sklearn's Multinomial Naive Bayes using the TF-IDF formula to convert our words to numbers.\n",
        "\n",
        "> 🔑**Note:** It's common practice to use non-DL algorithms as a baseline because of their speed and then later using DL to see if you can improve upon them."
      ],
      "metadata": {
        "id": "Z1QZhRUWpTuw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "# Create Tokenization and modelling pipeline\n",
        "model_0 = Pipeline([\n",
        "    (\"tfidf\", TfidfVectorizer()), # convert words to numbers using tfidf\n",
        "    (\"clf\", MultinomialNB()) # model the text\n",
        "])\n",
        "\n",
        "# Fit the pipeline to the training data\n",
        "model_0.fit(train_sentences, train_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "id": "lDO0r5NoqMta",
        "outputId": "00b95a62-2472-438c-9e76-e82536774cf2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('tfidf', TfidfVectorizer()), ('clf', MultinomialNB())])"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluate our baseline model\n",
        "baseline_score = model_0.score(val_sentences,val_labels)\n",
        "print(f\"Our baseline model achieves an accuracy of: {baseline_score*100:.2f}%\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AvYVRDjSsgVH",
        "outputId": "f0ea662b-9ff1-41a3-a685-a22cf90095bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Our baseline model achieves an accuracy of: 79.79%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions\n",
        "baseline_preds = model_0.predict(val_sentences)\n",
        "baseline_preds[:20]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "roYA-caYtApH",
        "outputId": "d9f009c6-686c-450e-bc80-509d7c5615da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Creating an evaluation function for our modelling experiments\n",
        "\n",
        "We could evaluate all of our model's predictions with different metrics everytime, however this will be cumbersome and could easily be fixed with a function.\n",
        "\n",
        "Let's create one to compare our model's predictions with the truth labels using the following metrics:\n",
        "* Accuracy\n",
        "* Precision\n",
        "* Recall\n",
        "* F1-score"
      ],
      "metadata": {
        "id": "jrx8_bGuuKim"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Function to evaluate: accuracy, precision, recall, f1-score\n",
        "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
        "\n",
        "def calculate_results(y_true,y_pred):\n",
        "  \"\"\"\n",
        "  Calculates model accuracy, precision, recall and f1 score of a binary classification model.\n",
        "  \"\"\"\n",
        "  # Calculate model accuracy\n",
        "  model_accuracy = accuracy_score(y_true,y_pred) * 100\n",
        "  # Calculate model precision, recall and f1-score using \"weighted\" average\n",
        "  model_precision, model_recall, model_f1, _ = precision_recall_fscore_support(y_true,y_pred,average=\"weighted\")\n",
        "  model_results = {\"accuracy\": model_accuracy,\n",
        "                   \"precision\": model_precision,\n",
        "                   \"recall\": model_recall,\n",
        "                   \"f1\": model_f1}\n",
        "  return model_results"
      ],
      "metadata": {
        "id": "_08wImElwOJc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get baseline results\n",
        "baseline_results = calculate_results(y_true=val_labels,\n",
        "                                     y_pred=baseline_preds)\n",
        "baseline_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cun72Obz02Is",
        "outputId": "ae77ce7e-3856-40e4-a0f1-61d965f3c293"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 79.79002624671917,\n",
              " 'precision': 0.8049118114708822,\n",
              " 'recall': 0.7979002624671916,\n",
              " 'f1': 0.7932923017133588}"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model 1: A simple dense model"
      ],
      "metadata": {
        "id": "17h8sxQP1Hu0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensorboard callback (need to create a new one for each model)\n",
        "from helper_functions import create_tensorboard_callback\n",
        "\n",
        "# Create a directory to save tensorboard logs\n",
        "SAVE_DIR = \"model_logs\""
      ],
      "metadata": {
        "id": "gZwt0UQPAZod"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Build model with the Functional API\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=tf.string) # inputs are 1-dimensional strings\n",
        "x = text_vectorizer(inputs) # turn the input text into numbers\n",
        "x = embedding(x) # create an embedding of the numerized inputs\n",
        "x = layers.GlobalAveragePooling1D()(x) # condense the feature vector for each token to one vector\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x) # create the output layer, want binary outputs so use sigmoid activation function\n",
        "model_1 = tf.keras.Model(inputs,outputs,name=\"model_1_dense\")"
      ],
      "metadata": {
        "id": "d3bhwuA_A_pw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_1.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J4D1NdZYCsR-",
        "outputId": "e22e0b23-2063-4eb0-d994-38dc6b12feaa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1_dense\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " global_average_pooling1d (  (None, 128)               0         \n",
            " GlobalAveragePooling1D)                                         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1280129 (4.88 MB)\n",
            "Trainable params: 1280129 (4.88 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile the model\n",
        "model_1.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "ic1nWEVCC2me"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model\n",
        "model_1_history = model_1.fit(x=train_sentences,\n",
        "                              y=train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(dir_name=SAVE_DIR,\n",
        "                                                                     experiment_name=\"model_1_dense\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fdiD8pJdDavV",
        "outputId": "774089c7-2d33-406a-83ea-23e62d991cff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_1_dense/20240826-160154\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 22s 90ms/step - loss: 0.6092 - accuracy: 0.7057 - val_loss: 0.5307 - val_accuracy: 0.7572\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 5s 22ms/step - loss: 0.4414 - accuracy: 0.8170 - val_loss: 0.4706 - val_accuracy: 0.7874\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 3s 12ms/step - loss: 0.3461 - accuracy: 0.8602 - val_loss: 0.4614 - val_accuracy: 0.7887\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 1s 5ms/step - loss: 0.2832 - accuracy: 0.8926 - val_loss: 0.4705 - val_accuracy: 0.7861\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 1s 5ms/step - loss: 0.2349 - accuracy: 0.9118 - val_loss: 0.4927 - val_accuracy: 0.7835\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the results\n",
        "model_1.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eydL3VQMD74l",
        "outputId": "70ba4570-0965-4b24-ad58-1a524e5f5928"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 0s 3ms/step - loss: 0.4927 - accuracy: 0.7835\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.49266135692596436, 0.7834645509719849]"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make some predictions and evaluate those\n",
        "model_1_pred_probs = model_1.predict(val_sentences)\n",
        "model_1_pred_probs[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3UEPzFHjFJ5V",
        "outputId": "2d9dee57-a775-4bce-869a-79eb5e30cc04"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 0s 2ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.9332523], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Looking at the first 10 predictions\n",
        "model_1_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wd6tlrLIFbdd",
        "outputId": "4fbb4706-22bb-4f3c-b6cb-a3f6cc276877"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.9332523 ],\n",
              "       [0.5176898 ],\n",
              "       [0.02935266],\n",
              "       [0.01690857],\n",
              "       [0.15615188],\n",
              "       [0.95781404],\n",
              "       [0.9635358 ],\n",
              "       [0.03265593],\n",
              "       [0.09347857],\n",
              "       [0.99869955]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert model predictions probabilities to label format\n",
        "model_1_preds = tf.squeeze(tf.round(model_1_pred_probs))\n",
        "model_1_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LHB1G3u6G_Xr",
        "outputId": "a930b76d-7712-4c53-cc0d-e2276480c390"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 1., 0., 0., 0., 1., 1., 0., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate our model_1 results\n",
        "model_1_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_1_preds)\n",
        "model_1_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4pSvZEI4HbQj",
        "outputId": "031d99df-fa67-4e2a-9697-966ca42188cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 78.34645669291339,\n",
              " 'precision': 0.7834899124733871,\n",
              " 'recall': 0.7834645669291339,\n",
              " 'f1': 0.7815717498014031}"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Checking our baseline results\n",
        "baseline_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "byx6CR7zHyAt",
        "outputId": "44d0b344-6441-4aef-efa2-ef98d1a0cf9d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 79.79002624671917,\n",
              " 'precision': 0.8049118114708822,\n",
              " 'recall': 0.7979002624671916,\n",
              " 'f1': 0.7932923017133588}"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Checking whether our deep learning model has better results than our baseline model or not\n",
        "import numpy as np\n",
        "np.array(list(model_1_results.values())) > np.array(list(baseline_results.values()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H1LVf-mBHoSE",
        "outputId": "76492026-bc32-4ad5-91b8-261a18c03d0a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([False, False, False, False])"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Visualizing learned embeddings"
      ],
      "metadata": {
        "id": "gysh8q2JIPAr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the vocabulary from the text vectorization layer\n",
        "words_in_vocab = text_vectorizer.get_vocabulary()\n",
        "len(words_in_vocab), words_in_vocab[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hIcaDi68L9FT",
        "outputId": "24dda466-f255-483b-b7e3-2a1d13a0202c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10000, ['', '[UNK]', 'the', 'a', 'to', 'in', 'of', 'and', 'i', 'is'])"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Model 1 summary\n",
        "model_1.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TtX8VN65MV8x",
        "outputId": "39ca8a95-9874-453f-89c2-71bf56fc909d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1_dense\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " global_average_pooling1d (  (None, 128)               0         \n",
            " GlobalAveragePooling1D)                                         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1280129 (4.88 MB)\n",
            "Trainable params: 1280129 (4.88 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the weight matrix of the embedding layer\n",
        "# (these are the numerical representations of each token in our training data, which have been learned for 5 epochs)\n",
        "embed_weights = model_1.get_layer(\"embedding\").get_weights()[0]\n",
        "embed_weights, embed_weights.shape # same size as vocab size and embedding_dim (output_dim of our embedding layer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b2An5T9gMfjy",
        "outputId": "2f135546-ca08-4b10-8bc6-98b1a5c9be9b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[-0.02944021, -0.02974595, -0.02869743, ...,  0.0485525 ,\n",
              "          0.01343261,  0.05261881],\n",
              "        [ 0.02543957, -0.00868237,  0.02968126, ...,  0.02044986,\n",
              "         -0.0354412 ,  0.01557357],\n",
              "        [ 0.02001975,  0.00896708,  0.00704742, ...,  0.05423174,\n",
              "          0.02786206,  0.02721484],\n",
              "        ...,\n",
              "        [ 0.09846632, -0.02381583, -0.01757441, ...,  0.02265988,\n",
              "          0.04527343,  0.09142945],\n",
              "        [ 0.01184744,  0.00159121, -0.04718976, ...,  0.01403471,\n",
              "          0.04407259,  0.03768918],\n",
              "        [-0.05426613,  0.03569142,  0.05401192, ..., -0.03051467,\n",
              "         -0.0842123 , -0.07765542]], dtype=float32),\n",
              " (10000, 128))"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we've got the embedding matrix our model has learned to represent our tokens, let's see how we can visualize it.\n",
        "\n",
        "To do so, TensorFlow has a handy tool called projector: https://projector.tensorflow.org/\n",
        "\n",
        "And TensorFlow also has an excellent guide on word embeddings themselves: https://www.tensorflow.org/tutorials/text/word_embeddings"
      ],
      "metadata": {
        "id": "PAM65D0iM6eJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create embedding files (we got this from TensorFlow's word embeddings documentation)\n",
        "import io\n",
        "out_v = io.open(\"vectors.tsv\", \"w\", encoding=\"utf-8\")\n",
        "out_m = io.open(\"metadata.tsv\", \"w\", encoding=\"utf-8\")\n",
        "\n",
        "for index, word in enumerate(words_in_vocab):\n",
        "  if index == 0:\n",
        "    continue # skip 0, it's padding\n",
        "  vec = embed_weights[index]\n",
        "  out_v.write(\"\\t\".join([str(x) for x in vec]) + \"\\n\")\n",
        "  out_m.write(word + \"\\n\")\n",
        "out_v.close()\n",
        "out_m.close()"
      ],
      "metadata": {
        "id": "MbhSk0WiPIx4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Download files from Colab to upload to projector\n",
        "try:\n",
        "  from google.colab import files\n",
        "  files.download(\"vectors.tsv\")\n",
        "  files.download(\"metadata.tsv\")\n",
        "except Exception:\n",
        "  pass"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "Jo_YEesOPjtI",
        "outputId": "86a1fb8d-3652-414d-d935-ec022c3c4867"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_1abd03b4-8331-4cbd-bdf1-ae73db3c4745\", \"vectors.tsv\", 15362891)"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_719f9c5f-72b2-4f5c-a379-53ee3dc27c6e\", \"metadata.tsv\", 80488)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Recurrent Neural Networks (RNN's)\n",
        "\n",
        "RNN's are useful for sequence data.\n",
        "\n",
        "The premise of a recurrent neural network is to use the representation of a previous input to aid the representation of a later input.\n",
        "\n",
        "**📖Resources:**\n",
        "If you want an overview of the internals of a recurrent neural network , see the following:\n",
        "- MIT's sequence modelling lecture: https://youtu.be/qjrad0V0uJE\n",
        "- Chris Olah's intro to LSTMs: https://colah.github.io/posts/2015-08-Understanding-LSTMs/\n",
        "- Andrej Karpathy's the unreasonable effectiveness of RNNs: http://karpathy.github.io/2015/05/21/rnn-effectiveness/\n"
      ],
      "metadata": {
        "id": "jgG0Q_I2QgIh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model 2: LSTM\n",
        "\n",
        "LSTM = Long Short Term Memory (one of the most popular LSTM cells)\n",
        "\n",
        "Our structure of an RNN typically looks like this :\n",
        "```\n",
        "Input(text) -> Tokenize -> Embedding -> Layers (RNNs/Dense) -> Output (label probability)\n",
        "```"
      ],
      "metadata": {
        "id": "D6zv8qljpfii"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create an LSTM model\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
        "x = text_vectorizer(inputs)\n",
        "x = embedding(x)\n",
        "#print(x.shape)\n",
        "#x = layers.LSTM(units=64, return_sequences=True)(x) # when you're stacking rnn cells together , you need to set return sequences=True\n",
        "#print(x.shape)\n",
        "x = layers.LSTM(64)(x)\n",
        "#print(x.shape)\n",
        "#x = layers.Dense(64, activation=\"relu\")(x)\n",
        "#print(x.shape)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_2 = tf.keras.Model(inputs,outputs,name = \"model_2_LSTM\")"
      ],
      "metadata": {
        "id": "dWGNcY1_qqIh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a summary of our model 2\n",
        "model_2.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q32LtulbsAmJ",
        "outputId": "9c02764d-298b-4c3b-af3a-f14d1282d866"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_2_LSTM\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " lstm (LSTM)                 (None, 64)                49408     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1329473 (5.07 MB)\n",
            "Trainable params: 1329473 (5.07 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile the model\n",
        "model_2.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "ERxjlbGPuJtG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model\n",
        "model_2_history = model_2.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
        "                                                                     \"model_2_LSTM\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YBbb9IVTukTl",
        "outputId": "4200a727-5ff5-478d-e6ba-02f6d0d4f3b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_2_LSTM/20240826-160239\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 11s 37ms/step - loss: 0.2157 - accuracy: 0.9237 - val_loss: 0.6109 - val_accuracy: 0.7612\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 2s 9ms/step - loss: 0.1521 - accuracy: 0.9432 - val_loss: 0.7292 - val_accuracy: 0.7572\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 3s 12ms/step - loss: 0.1266 - accuracy: 0.9542 - val_loss: 0.7395 - val_accuracy: 0.7388\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 2s 10ms/step - loss: 0.1013 - accuracy: 0.9637 - val_loss: 0.8612 - val_accuracy: 0.7415\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.0847 - accuracy: 0.9672 - val_loss: 1.0160 - val_accuracy: 0.7362\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions with LSTM's model\n",
        "model_2_pred_probs = model_2.predict(val_sentences)\n",
        "model_2_pred_probs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9xGFYCKCvZtG",
        "outputId": "bbb4dab7-2a9f-432f-d197-a04a093f76f3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 0s 3ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[9.74485755e-01],\n",
              "       [3.41281682e-01],\n",
              "       [2.07626028e-03],\n",
              "       [7.56050867e-04],\n",
              "       [2.46443450e-01],\n",
              "       [9.99582469e-01],\n",
              "       [9.99444425e-01],\n",
              "       [7.24329962e-04],\n",
              "       [5.83686400e-04],\n",
              "       [9.99621868e-01],\n",
              "       [4.79295030e-02],\n",
              "       [3.87503207e-03],\n",
              "       [9.99112308e-01],\n",
              "       [2.88668163e-02],\n",
              "       [1.46989241e-01],\n",
              "       [2.43674517e-02],\n",
              "       [1.06075974e-02],\n",
              "       [1.99650065e-03],\n",
              "       [8.72637182e-02],\n",
              "       [4.06031966e-01],\n",
              "       [1.15894988e-01],\n",
              "       [9.99246716e-01],\n",
              "       [1.82601377e-01],\n",
              "       [9.60512280e-01],\n",
              "       [9.99822319e-01],\n",
              "       [2.46453762e-01],\n",
              "       [2.60467903e-04],\n",
              "       [9.99500990e-01],\n",
              "       [2.18813494e-03],\n",
              "       [9.98732388e-01],\n",
              "       [8.40608205e-04],\n",
              "       [9.99039292e-01],\n",
              "       [2.86458852e-03],\n",
              "       [8.85849059e-01],\n",
              "       [9.89998996e-01],\n",
              "       [4.46938537e-03],\n",
              "       [6.11255109e-01],\n",
              "       [8.26885551e-02],\n",
              "       [1.67559814e-02],\n",
              "       [1.08398348e-02],\n",
              "       [4.70920622e-01],\n",
              "       [9.99647021e-01],\n",
              "       [9.86725569e-01],\n",
              "       [1.10932931e-01],\n",
              "       [4.95740660e-02],\n",
              "       [6.69863671e-02],\n",
              "       [6.52143359e-03],\n",
              "       [8.49466946e-04],\n",
              "       [2.73784697e-01],\n",
              "       [5.43915387e-03],\n",
              "       [8.74692202e-03],\n",
              "       [2.42039387e-04],\n",
              "       [1.83217449e-03],\n",
              "       [9.99397278e-01],\n",
              "       [9.98793483e-01],\n",
              "       [3.65561187e-01],\n",
              "       [9.99350965e-01],\n",
              "       [9.99441206e-01],\n",
              "       [9.99677896e-01],\n",
              "       [8.57408285e-01],\n",
              "       [2.65749753e-01],\n",
              "       [9.69627142e-01],\n",
              "       [6.02012314e-02],\n",
              "       [8.91542342e-03],\n",
              "       [2.94172838e-02],\n",
              "       [9.99564826e-01],\n",
              "       [6.02992363e-02],\n",
              "       [2.24697357e-03],\n",
              "       [9.97534275e-01],\n",
              "       [2.69581318e-01],\n",
              "       [3.10734838e-01],\n",
              "       [2.97520938e-03],\n",
              "       [2.59827465e-01],\n",
              "       [9.37580526e-01],\n",
              "       [2.28685036e-01],\n",
              "       [4.46615828e-04],\n",
              "       [5.39779896e-03],\n",
              "       [2.54254878e-01],\n",
              "       [9.68331933e-01],\n",
              "       [1.88428245e-03],\n",
              "       [9.89015758e-01],\n",
              "       [3.85756344e-02],\n",
              "       [9.99683499e-01],\n",
              "       [4.07609157e-03],\n",
              "       [9.70739007e-01],\n",
              "       [3.72048467e-01],\n",
              "       [5.84949739e-03],\n",
              "       [5.16706938e-03],\n",
              "       [9.99604166e-01],\n",
              "       [5.71019530e-01],\n",
              "       [9.99698639e-01],\n",
              "       [2.03910433e-02],\n",
              "       [3.93714756e-03],\n",
              "       [9.99618053e-01],\n",
              "       [9.96820092e-01],\n",
              "       [9.98178601e-01],\n",
              "       [9.99626875e-01],\n",
              "       [2.54818541e-03],\n",
              "       [6.11220603e-04],\n",
              "       [4.49643889e-03],\n",
              "       [9.98985827e-01],\n",
              "       [9.99624133e-01],\n",
              "       [9.99622226e-01],\n",
              "       [1.96308538e-01],\n",
              "       [5.26492894e-02],\n",
              "       [5.79968281e-03],\n",
              "       [9.05916095e-01],\n",
              "       [2.27822036e-01],\n",
              "       [9.37850952e-01],\n",
              "       [9.15693585e-04],\n",
              "       [2.34439950e-02],\n",
              "       [9.99550641e-01],\n",
              "       [9.99542594e-01],\n",
              "       [9.99541163e-01],\n",
              "       [9.12828267e-01],\n",
              "       [5.08255325e-04],\n",
              "       [9.64323874e-04],\n",
              "       [9.99589622e-01],\n",
              "       [7.57418515e-04],\n",
              "       [1.42233610e-01],\n",
              "       [1.61335599e-02],\n",
              "       [1.63840759e-03],\n",
              "       [9.99516606e-01],\n",
              "       [6.68389052e-02],\n",
              "       [9.99635696e-01],\n",
              "       [9.99230623e-01],\n",
              "       [1.84749827e-01],\n",
              "       [7.84232107e-04],\n",
              "       [8.52192636e-04],\n",
              "       [9.99764979e-01],\n",
              "       [1.86903365e-02],\n",
              "       [2.83800560e-04],\n",
              "       [9.98057306e-01],\n",
              "       [4.66601318e-03],\n",
              "       [3.70802591e-04],\n",
              "       [1.87853426e-01],\n",
              "       [9.24577191e-02],\n",
              "       [7.81183839e-02],\n",
              "       [9.99712169e-01],\n",
              "       [9.98578668e-01],\n",
              "       [9.65605736e-01],\n",
              "       [9.99797881e-01],\n",
              "       [6.85398292e-04],\n",
              "       [3.98256853e-02],\n",
              "       [5.95335007e-01],\n",
              "       [3.90458465e-01],\n",
              "       [9.99647021e-01],\n",
              "       [9.96515036e-01],\n",
              "       [9.99838233e-01],\n",
              "       [9.26065803e-01],\n",
              "       [5.98870367e-02],\n",
              "       [9.99589384e-01],\n",
              "       [5.98906481e-04],\n",
              "       [8.66680264e-01],\n",
              "       [2.60909885e-01],\n",
              "       [9.99818861e-01],\n",
              "       [9.99829650e-01],\n",
              "       [7.17166159e-03],\n",
              "       [6.73065558e-02],\n",
              "       [3.84468539e-03],\n",
              "       [2.65749753e-01],\n",
              "       [1.07393588e-03],\n",
              "       [7.83845410e-02],\n",
              "       [9.99596894e-01],\n",
              "       [1.24458484e-02],\n",
              "       [2.72562534e-01],\n",
              "       [2.09662661e-01],\n",
              "       [9.99886632e-01],\n",
              "       [9.99591410e-01],\n",
              "       [9.99524236e-01],\n",
              "       [9.76304352e-01],\n",
              "       [9.97901320e-01],\n",
              "       [1.73890730e-03],\n",
              "       [3.25688161e-04],\n",
              "       [2.16564819e-01],\n",
              "       [7.05230176e-01],\n",
              "       [9.98997152e-01],\n",
              "       [1.11559336e-03],\n",
              "       [9.34367120e-01],\n",
              "       [7.13635623e-01],\n",
              "       [9.98245358e-01],\n",
              "       [3.54091870e-04],\n",
              "       [9.98818934e-01],\n",
              "       [3.03431153e-02],\n",
              "       [2.98630656e-03],\n",
              "       [9.99774039e-01],\n",
              "       [4.98287240e-03],\n",
              "       [1.65402964e-01],\n",
              "       [1.21343601e-02],\n",
              "       [9.98959422e-01],\n",
              "       [2.32467949e-01],\n",
              "       [9.99521017e-01],\n",
              "       [2.74091452e-01],\n",
              "       [1.55318547e-02],\n",
              "       [3.94469917e-01],\n",
              "       [4.01004654e-04],\n",
              "       [1.23823935e-03],\n",
              "       [6.33386779e-04],\n",
              "       [1.16783660e-02],\n",
              "       [9.84985590e-01],\n",
              "       [1.28822541e-03],\n",
              "       [9.99267399e-01],\n",
              "       [9.99860048e-01],\n",
              "       [9.99177158e-01],\n",
              "       [9.96037602e-01],\n",
              "       [2.72128247e-02],\n",
              "       [2.60808498e-01],\n",
              "       [4.45834128e-03],\n",
              "       [9.99276459e-01],\n",
              "       [9.87701118e-01],\n",
              "       [2.36439958e-01],\n",
              "       [1.75308853e-01],\n",
              "       [5.61672216e-03],\n",
              "       [1.96873373e-03],\n",
              "       [2.65749753e-01],\n",
              "       [3.46767105e-04],\n",
              "       [2.75777304e-03],\n",
              "       [3.22955172e-03],\n",
              "       [3.46767105e-04],\n",
              "       [9.99162555e-01],\n",
              "       [7.89516978e-03],\n",
              "       [1.37321989e-03],\n",
              "       [9.99494910e-01],\n",
              "       [8.77665728e-03],\n",
              "       [1.63151801e-01],\n",
              "       [1.40371302e-03],\n",
              "       [4.34879126e-04],\n",
              "       [1.85321213e-03],\n",
              "       [1.18547014e-03],\n",
              "       [9.99392867e-01],\n",
              "       [1.06857088e-03],\n",
              "       [1.06662242e-02],\n",
              "       [9.87269402e-01],\n",
              "       [1.93990752e-01],\n",
              "       [3.26056615e-04],\n",
              "       [8.68716165e-02],\n",
              "       [9.98945892e-01],\n",
              "       [9.73887682e-01],\n",
              "       [8.25882554e-01],\n",
              "       [9.99653935e-01],\n",
              "       [1.99703410e-01],\n",
              "       [1.95988715e-02],\n",
              "       [7.85282195e-01],\n",
              "       [7.03042885e-03],\n",
              "       [9.97449696e-01],\n",
              "       [9.31758821e-01],\n",
              "       [1.20274395e-01],\n",
              "       [9.99403715e-01],\n",
              "       [9.99703586e-01],\n",
              "       [9.52345669e-01],\n",
              "       [9.99377310e-01],\n",
              "       [9.99852300e-01],\n",
              "       [9.99072313e-01],\n",
              "       [2.42731541e-01],\n",
              "       [3.14714536e-02],\n",
              "       [2.21848721e-03],\n",
              "       [4.18717414e-03],\n",
              "       [3.60772729e-01],\n",
              "       [6.46581233e-04],\n",
              "       [4.12775278e-02],\n",
              "       [2.67909646e-01],\n",
              "       [1.54707478e-02],\n",
              "       [2.14257106e-01],\n",
              "       [9.98973489e-01],\n",
              "       [7.91897848e-02],\n",
              "       [8.43585120e-04],\n",
              "       [8.47390294e-02],\n",
              "       [3.63937870e-04],\n",
              "       [7.26295948e-01],\n",
              "       [9.99212027e-01],\n",
              "       [1.24731213e-01],\n",
              "       [9.99706924e-01],\n",
              "       [9.99770820e-01],\n",
              "       [2.69371778e-01],\n",
              "       [9.75393236e-01],\n",
              "       [9.43625629e-01],\n",
              "       [9.99583304e-01],\n",
              "       [5.28152799e-04],\n",
              "       [9.99322057e-01],\n",
              "       [5.37800848e-01],\n",
              "       [9.99672174e-01],\n",
              "       [9.99840140e-01],\n",
              "       [1.47517174e-01],\n",
              "       [9.99581873e-01],\n",
              "       [9.99558032e-01],\n",
              "       [8.69293451e-01],\n",
              "       [9.97170150e-01],\n",
              "       [2.32484117e-01],\n",
              "       [9.99368846e-01],\n",
              "       [7.63913453e-01],\n",
              "       [9.99537349e-01],\n",
              "       [4.54364002e-01],\n",
              "       [1.19625460e-02],\n",
              "       [1.07080408e-03],\n",
              "       [8.33789855e-02],\n",
              "       [3.80903073e-02],\n",
              "       [9.99840260e-01],\n",
              "       [2.77522430e-02],\n",
              "       [1.24313282e-02],\n",
              "       [9.99354184e-01],\n",
              "       [9.99688506e-01],\n",
              "       [9.99446094e-01],\n",
              "       [9.99282777e-01],\n",
              "       [2.56719381e-01],\n",
              "       [2.07626028e-03],\n",
              "       [5.23168186e-04],\n",
              "       [9.99663115e-01],\n",
              "       [3.57686821e-03],\n",
              "       [9.79170669e-04],\n",
              "       [2.65749753e-01],\n",
              "       [9.99573171e-01],\n",
              "       [9.99533534e-01],\n",
              "       [2.03160569e-01],\n",
              "       [9.98833358e-01],\n",
              "       [5.37942676e-03],\n",
              "       [9.99599636e-01],\n",
              "       [9.98562396e-01],\n",
              "       [2.81329895e-03],\n",
              "       [4.26811428e-04],\n",
              "       [5.27235746e-01],\n",
              "       [9.35588717e-01],\n",
              "       [2.28081662e-02],\n",
              "       [9.38809197e-03],\n",
              "       [9.99841094e-01],\n",
              "       [9.99714434e-01],\n",
              "       [9.99399662e-01],\n",
              "       [5.37585095e-03],\n",
              "       [7.99462851e-03],\n",
              "       [9.99900222e-01],\n",
              "       [9.89009321e-01],\n",
              "       [9.99694705e-01],\n",
              "       [6.95777059e-01],\n",
              "       [2.75758833e-01],\n",
              "       [9.95893478e-01],\n",
              "       [2.04606005e-03],\n",
              "       [9.99595463e-01],\n",
              "       [2.26885185e-01],\n",
              "       [4.72923741e-03],\n",
              "       [1.38664804e-03],\n",
              "       [5.28863072e-02],\n",
              "       [4.98514739e-04],\n",
              "       [9.95845258e-01],\n",
              "       [1.64431036e-01],\n",
              "       [9.98582363e-01],\n",
              "       [1.41569916e-02],\n",
              "       [4.84245858e-04],\n",
              "       [9.12531257e-01],\n",
              "       [8.80832732e-01],\n",
              "       [3.12171155e-03],\n",
              "       [9.98661518e-01],\n",
              "       [9.98992145e-01],\n",
              "       [1.21212529e-03],\n",
              "       [3.20591450e-01],\n",
              "       [1.55191377e-01],\n",
              "       [4.19400109e-04],\n",
              "       [5.40630752e-03],\n",
              "       [5.41826822e-02],\n",
              "       [1.36703691e-02],\n",
              "       [9.99524236e-01],\n",
              "       [9.99691963e-01],\n",
              "       [4.40166853e-02],\n",
              "       [9.98430550e-01],\n",
              "       [9.79826331e-01],\n",
              "       [9.04831052e-01],\n",
              "       [9.94548261e-01],\n",
              "       [9.99186933e-01],\n",
              "       [7.31806815e-01],\n",
              "       [8.26522650e-04],\n",
              "       [9.99636173e-01],\n",
              "       [1.27179094e-03],\n",
              "       [5.61830448e-03],\n",
              "       [7.56894588e-01],\n",
              "       [1.18470125e-01],\n",
              "       [9.92568493e-01],\n",
              "       [5.02113402e-02],\n",
              "       [9.99817908e-01],\n",
              "       [2.67209462e-03],\n",
              "       [2.22657584e-02],\n",
              "       [9.00344166e-04],\n",
              "       [4.78482014e-03],\n",
              "       [3.82587430e-04],\n",
              "       [8.90535295e-01],\n",
              "       [6.20686635e-03],\n",
              "       [2.70592839e-01],\n",
              "       [9.99054611e-01],\n",
              "       [7.14678168e-02],\n",
              "       [1.99272968e-02],\n",
              "       [3.41601044e-01],\n",
              "       [2.18434026e-03],\n",
              "       [4.16036375e-04],\n",
              "       [9.99642372e-01],\n",
              "       [1.09931985e-02],\n",
              "       [9.12247837e-01],\n",
              "       [9.89578545e-01],\n",
              "       [7.83808471e-04],\n",
              "       [9.98777989e-03],\n",
              "       [4.54892628e-02],\n",
              "       [9.88775611e-01],\n",
              "       [8.04995594e-04],\n",
              "       [1.38737604e-01],\n",
              "       [9.93185639e-01],\n",
              "       [1.19529583e-01],\n",
              "       [4.62317257e-04],\n",
              "       [9.80630875e-01],\n",
              "       [9.99183834e-01],\n",
              "       [2.34758970e-03],\n",
              "       [4.62065043e-04],\n",
              "       [5.82616143e-02],\n",
              "       [5.80722420e-03],\n",
              "       [9.99407768e-01],\n",
              "       [3.48600443e-04],\n",
              "       [6.55878007e-01],\n",
              "       [2.78553995e-03],\n",
              "       [1.88088313e-01],\n",
              "       [6.93445420e-03],\n",
              "       [9.97453749e-01],\n",
              "       [2.29103733e-02],\n",
              "       [1.02620421e-03],\n",
              "       [1.06480457e-01],\n",
              "       [9.99811232e-01],\n",
              "       [3.06555332e-04],\n",
              "       [9.56101179e-01],\n",
              "       [9.81749967e-03],\n",
              "       [9.99636173e-01],\n",
              "       [3.95893380e-02],\n",
              "       [7.85166681e-01],\n",
              "       [5.88759140e-04],\n",
              "       [9.99703944e-01],\n",
              "       [3.95855837e-04],\n",
              "       [4.97023799e-02],\n",
              "       [1.49695412e-03],\n",
              "       [7.39899158e-01],\n",
              "       [9.96433973e-01],\n",
              "       [9.99836206e-01],\n",
              "       [2.05998332e-03],\n",
              "       [1.40710156e-02],\n",
              "       [9.99101996e-01],\n",
              "       [9.95845258e-01],\n",
              "       [9.80353117e-01],\n",
              "       [9.99636173e-01],\n",
              "       [2.26322678e-03],\n",
              "       [6.07927382e-01],\n",
              "       [9.51985456e-03],\n",
              "       [7.09740296e-02],\n",
              "       [9.87379134e-01],\n",
              "       [9.98782575e-01],\n",
              "       [5.10640536e-03],\n",
              "       [8.42743516e-01],\n",
              "       [9.37744141e-01],\n",
              "       [2.42257286e-02],\n",
              "       [9.99861360e-01],\n",
              "       [4.30875212e-01],\n",
              "       [2.00082716e-02],\n",
              "       [9.96715069e-01],\n",
              "       [1.31066022e-02],\n",
              "       [1.37006976e-02],\n",
              "       [9.99797881e-01],\n",
              "       [9.99578774e-01],\n",
              "       [9.99446809e-01],\n",
              "       [4.18727333e-03],\n",
              "       [2.85920501e-01],\n",
              "       [9.99311566e-01],\n",
              "       [2.60265041e-02],\n",
              "       [5.19981198e-02],\n",
              "       [1.57183200e-01],\n",
              "       [9.99714434e-01],\n",
              "       [7.39648298e-04],\n",
              "       [2.63602048e-01],\n",
              "       [1.01798408e-01],\n",
              "       [2.50461325e-02],\n",
              "       [1.55898568e-03],\n",
              "       [9.99618053e-01],\n",
              "       [9.99602139e-01],\n",
              "       [9.43625629e-01],\n",
              "       [9.99719679e-01],\n",
              "       [1.56392545e-01],\n",
              "       [2.61851167e-03],\n",
              "       [1.09995663e-01],\n",
              "       [3.00541031e-03],\n",
              "       [9.99647021e-01],\n",
              "       [3.59179149e-03],\n",
              "       [1.80815868e-02],\n",
              "       [5.55624627e-02],\n",
              "       [9.99661207e-01],\n",
              "       [9.99426961e-01],\n",
              "       [5.79991937e-03],\n",
              "       [8.13301921e-01],\n",
              "       [6.24053180e-03],\n",
              "       [9.99622941e-01],\n",
              "       [1.07204355e-01],\n",
              "       [9.99894738e-01],\n",
              "       [5.08283905e-04],\n",
              "       [1.03359437e-03],\n",
              "       [3.74916987e-03],\n",
              "       [9.99203026e-01],\n",
              "       [1.59820896e-02],\n",
              "       [4.35850143e-01],\n",
              "       [7.91897848e-02],\n",
              "       [9.98948753e-01],\n",
              "       [9.99792874e-01],\n",
              "       [1.18034100e-03],\n",
              "       [2.19174504e-01],\n",
              "       [4.16345261e-02],\n",
              "       [3.56172442e-01],\n",
              "       [4.62774711e-04],\n",
              "       [4.38082702e-02],\n",
              "       [2.62593776e-01],\n",
              "       [6.07259572e-03],\n",
              "       [9.68596876e-01],\n",
              "       [9.99183953e-01],\n",
              "       [9.66096461e-01],\n",
              "       [3.07698883e-02],\n",
              "       [1.18513675e-02],\n",
              "       [3.59719340e-03],\n",
              "       [1.56336208e-03],\n",
              "       [5.63011598e-03],\n",
              "       [9.91481662e-01],\n",
              "       [9.99865294e-01],\n",
              "       [9.99449670e-01],\n",
              "       [2.97915250e-01],\n",
              "       [9.35211420e-01],\n",
              "       [1.78507790e-02],\n",
              "       [5.52135461e-04],\n",
              "       [4.25986061e-03],\n",
              "       [8.88645742e-03],\n",
              "       [4.62095020e-03],\n",
              "       [6.97722833e-04],\n",
              "       [1.27820680e-02],\n",
              "       [6.32920146e-01],\n",
              "       [1.87927727e-02],\n",
              "       [3.42384446e-03],\n",
              "       [1.25376629e-02],\n",
              "       [5.54016908e-04],\n",
              "       [1.12339310e-01],\n",
              "       [9.86725569e-01],\n",
              "       [9.99755681e-01],\n",
              "       [9.98425961e-01],\n",
              "       [1.33734057e-02],\n",
              "       [9.99728739e-01],\n",
              "       [3.98523081e-03],\n",
              "       [1.31072234e-02],\n",
              "       [9.99527335e-01],\n",
              "       [7.96158449e-04],\n",
              "       [9.99524236e-01],\n",
              "       [4.73706564e-03],\n",
              "       [2.95851558e-01],\n",
              "       [4.05482687e-02],\n",
              "       [2.56760027e-02],\n",
              "       [9.53454196e-01],\n",
              "       [5.34470797e-01],\n",
              "       [9.99646306e-01],\n",
              "       [7.31173949e-03],\n",
              "       [1.01133892e-02],\n",
              "       [9.99369323e-01],\n",
              "       [1.62688200e-03],\n",
              "       [6.08800119e-03],\n",
              "       [8.10881317e-01],\n",
              "       [9.00750805e-04],\n",
              "       [9.99552310e-01],\n",
              "       [9.95418310e-01],\n",
              "       [4.19761278e-02],\n",
              "       [9.84067976e-01],\n",
              "       [9.99457181e-01],\n",
              "       [7.47515238e-04],\n",
              "       [8.27567594e-04],\n",
              "       [3.44360620e-03],\n",
              "       [8.64020269e-03],\n",
              "       [7.77062058e-01],\n",
              "       [9.96696949e-01],\n",
              "       [1.67797834e-01],\n",
              "       [9.99894977e-01],\n",
              "       [2.59931287e-04],\n",
              "       [9.99352753e-01],\n",
              "       [9.96390522e-01],\n",
              "       [9.22436416e-01],\n",
              "       [5.17284155e-01],\n",
              "       [8.56125951e-01],\n",
              "       [3.20062623e-04],\n",
              "       [2.59827465e-01],\n",
              "       [1.82030722e-02],\n",
              "       [3.62232124e-04],\n",
              "       [1.08226109e-03],\n",
              "       [5.49671182e-04],\n",
              "       [8.92558634e-01],\n",
              "       [9.94913340e-01],\n",
              "       [8.79448652e-01],\n",
              "       [3.05478442e-02],\n",
              "       [2.31864769e-03],\n",
              "       [9.76022840e-01],\n",
              "       [4.48076934e-01],\n",
              "       [9.10649061e-01],\n",
              "       [9.73778889e-02],\n",
              "       [1.73922279e-03],\n",
              "       [1.10797547e-02],\n",
              "       [4.98539943e-04],\n",
              "       [9.98914719e-01],\n",
              "       [2.05006525e-01],\n",
              "       [6.23128079e-02],\n",
              "       [9.99876857e-01],\n",
              "       [9.99267399e-01],\n",
              "       [8.16156864e-01],\n",
              "       [4.66929972e-01],\n",
              "       [9.98801827e-01],\n",
              "       [2.99234651e-02],\n",
              "       [8.12837958e-01],\n",
              "       [1.51242188e-03],\n",
              "       [3.60322624e-01],\n",
              "       [2.31360458e-03],\n",
              "       [9.94656444e-01],\n",
              "       [7.94393290e-03],\n",
              "       [6.78986893e-04],\n",
              "       [9.77673173e-01],\n",
              "       [2.07308074e-03],\n",
              "       [3.77170363e-04],\n",
              "       [4.42195684e-01],\n",
              "       [6.71515405e-01],\n",
              "       [4.05208394e-03],\n",
              "       [9.23100160e-04],\n",
              "       [2.03886449e-01],\n",
              "       [4.19484377e-02],\n",
              "       [4.99652438e-02],\n",
              "       [9.97232974e-01],\n",
              "       [2.02509582e-01],\n",
              "       [3.39821763e-02],\n",
              "       [8.97533536e-01],\n",
              "       [4.30833884e-02],\n",
              "       [4.95938351e-04],\n",
              "       [9.99019742e-01],\n",
              "       [9.99521613e-01],\n",
              "       [4.36055303e-01],\n",
              "       [1.53471716e-03],\n",
              "       [3.38379174e-01],\n",
              "       [1.09377829e-03],\n",
              "       [9.35588717e-01],\n",
              "       [1.23288818e-02],\n",
              "       [5.06816357e-02],\n",
              "       [3.15191108e-03],\n",
              "       [9.99407887e-01],\n",
              "       [3.21597785e-01],\n",
              "       [9.95488465e-01],\n",
              "       [9.98991668e-01],\n",
              "       [7.69733131e-01],\n",
              "       [2.94821383e-03],\n",
              "       [9.99647975e-01],\n",
              "       [9.99789894e-01],\n",
              "       [2.55974446e-04],\n",
              "       [9.53970291e-03],\n",
              "       [4.71759995e-04],\n",
              "       [9.99783695e-01],\n",
              "       [9.99871016e-01],\n",
              "       [9.26126540e-01],\n",
              "       [9.99662399e-01],\n",
              "       [9.99698639e-01],\n",
              "       [7.55922217e-03],\n",
              "       [9.91468668e-01],\n",
              "       [5.74828405e-03],\n",
              "       [9.03188825e-01],\n",
              "       [9.99241829e-01],\n",
              "       [8.76894977e-04],\n",
              "       [3.08735622e-03],\n",
              "       [9.99724329e-01],\n",
              "       [9.51207459e-01],\n",
              "       [9.95881438e-01],\n",
              "       [8.78224820e-02],\n",
              "       [9.99554098e-01],\n",
              "       [2.26413575e-03],\n",
              "       [9.98894393e-01],\n",
              "       [1.23971239e-01],\n",
              "       [9.99752462e-01],\n",
              "       [7.41320908e-01],\n",
              "       [4.88218665e-03],\n",
              "       [9.99679327e-01],\n",
              "       [9.99574125e-01],\n",
              "       [3.46410117e-04],\n",
              "       [9.99797642e-01],\n",
              "       [9.91390824e-01],\n",
              "       [3.14493239e-01],\n",
              "       [4.03795857e-03],\n",
              "       [9.99485016e-01],\n",
              "       [9.98079181e-01],\n",
              "       [7.63978763e-03],\n",
              "       [9.99813259e-01],\n",
              "       [9.99694705e-01],\n",
              "       [1.24686130e-03],\n",
              "       [9.95070338e-02],\n",
              "       [1.51065155e-03],\n",
              "       [8.36993039e-01],\n",
              "       [9.62657809e-01],\n",
              "       [2.09416933e-02],\n",
              "       [9.99684691e-01],\n",
              "       [9.99795616e-01],\n",
              "       [6.12503896e-03],\n",
              "       [9.99629498e-01],\n",
              "       [3.08169603e-01],\n",
              "       [9.76380467e-01],\n",
              "       [9.99472082e-01],\n",
              "       [9.90089536e-01],\n",
              "       [9.36648726e-01],\n",
              "       [2.91444798e-04],\n",
              "       [2.03123642e-03],\n",
              "       [3.13561589e-01],\n",
              "       [9.99626875e-01],\n",
              "       [2.80785956e-04],\n",
              "       [9.98886764e-01],\n",
              "       [9.97625768e-01],\n",
              "       [1.53070083e-02],\n",
              "       [9.41199720e-01],\n",
              "       [9.99494910e-01],\n",
              "       [9.99699354e-01],\n",
              "       [1.26507301e-02],\n",
              "       [5.99300535e-03],\n",
              "       [1.72934085e-02],\n",
              "       [9.95491683e-01],\n",
              "       [3.79225820e-01],\n",
              "       [9.96832788e-01],\n",
              "       [3.52406204e-02],\n",
              "       [9.99289751e-01],\n",
              "       [1.53551891e-03],\n",
              "       [9.85927656e-02],\n",
              "       [1.48258597e-01],\n",
              "       [1.34211185e-03],\n",
              "       [9.99370873e-01],\n",
              "       [1.21244229e-01],\n",
              "       [4.02929857e-02],\n",
              "       [1.99922264e-01],\n",
              "       [9.99810994e-01],\n",
              "       [9.99039829e-01],\n",
              "       [4.22047917e-04],\n",
              "       [3.77034489e-03],\n",
              "       [1.52248023e-02],\n",
              "       [9.84749675e-01],\n",
              "       [9.96046126e-01],\n",
              "       [1.28868073e-01],\n",
              "       [9.99277413e-01],\n",
              "       [1.99905247e-03],\n",
              "       [3.07928829e-04],\n",
              "       [9.99626040e-01],\n",
              "       [4.07623760e-02],\n",
              "       [9.99474466e-01],\n",
              "       [1.40071493e-02],\n",
              "       [9.67699230e-01],\n",
              "       [9.66625381e-03],\n",
              "       [8.11094940e-01],\n",
              "       [9.99536157e-01],\n",
              "       [2.19473951e-02],\n",
              "       [5.05898055e-03],\n",
              "       [8.55668783e-01],\n",
              "       [9.99542594e-01],\n",
              "       [2.78010294e-02],\n",
              "       [3.61837144e-03],\n",
              "       [5.74729621e-01],\n",
              "       [2.72216164e-02],\n",
              "       [1.24437101e-02],\n",
              "       [9.99848127e-01],\n",
              "       [4.55420883e-03],\n",
              "       [3.91499314e-04],\n",
              "       [9.65852082e-01],\n",
              "       [2.29456685e-02],\n",
              "       [1.87100540e-03],\n",
              "       [2.15167791e-04],\n",
              "       [7.64055690e-03],\n",
              "       [9.99724209e-01]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert model 2 pred probs to labels\n",
        "model_2_preds = tf.squeeze(tf.round(model_2_pred_probs))\n",
        "model_2_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5hxynUz3vrgk",
        "outputId": "c05d397a-eea5-4c15-a72f-9424813f2dd1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 0., 0., 0., 0., 1., 1., 0., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate model_2 results\n",
        "model_2_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_2_preds)\n",
        "model_2_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "By9e9GCrv_v8",
        "outputId": "ccff95c0-dec5-4c46-db6a-e06074a64b8f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 73.62204724409449,\n",
              " 'precision': 0.7356783499799904,\n",
              " 'recall': 0.7362204724409449,\n",
              " 'f1': 0.7358868349636495}"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can still see that our RNN layer using LSTM doesn't beat our baseline model"
      ],
      "metadata": {
        "id": "hEDhs9qDwTKN"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model 3 : GRU\n",
        "\n",
        "Another popular and effective RNN component is the GRU or gated recurrent unit.\n",
        "\n",
        "The GRU cell has similar features to an LSTM cell but has less parameters."
      ],
      "metadata": {
        "id": "hzfW4pIvwvDl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Build an RNN using the GRU cell\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=tf.string)\n",
        "x = text_vectorizer(inputs)\n",
        "x = embedding(x)\n",
        "x = layers.GRU(64)(x)\n",
        "#x = layers.LSTM(64)(x)\n",
        "#x = layers.GRU(64)(x)\n",
        "#x = layers.Dense(64, activation=\"relu\")(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_3 = tf.keras.Model(inputs,outputs, name=\"model_3_GRU\")"
      ],
      "metadata": {
        "id": "0WcfkmZlxxRo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_3.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QtrdY8Fd00ob",
        "outputId": "6334505b-b15b-4a1b-84de-250dc2710bd1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_3_GRU\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_3 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " gru (GRU)                   (None, 64)                37248     \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1317313 (5.03 MB)\n",
            "Trainable params: 1317313 (5.03 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile the model\n",
        "model_3.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "B5k5Mvoi3foq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model\n",
        "model_3_history = model_3.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
        "                                                                     \"model_3_GRU\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eiolpKGH33j5",
        "outputId": "9f4872c4-411e-41c0-d6d1-0d4363123904"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_3_GRU/20240826-160300\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 11s 40ms/step - loss: 0.1556 - accuracy: 0.9410 - val_loss: 0.8924 - val_accuracy: 0.7388\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 2s 10ms/step - loss: 0.0816 - accuracy: 0.9715 - val_loss: 0.8376 - val_accuracy: 0.7520\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 2s 10ms/step - loss: 0.0717 - accuracy: 0.9736 - val_loss: 1.1107 - val_accuracy: 0.7323\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.0590 - accuracy: 0.9771 - val_loss: 1.1586 - val_accuracy: 0.7270\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.0529 - accuracy: 0.9766 - val_loss: 1.3298 - val_accuracy: 0.7310\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make some predictions with our GRU model\n",
        "model_3_pred_probs = model_3.predict(val_sentences)\n",
        "model_3_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GLmotIHD4ZJZ",
        "outputId": "fbe2af61-09de-4d14-b568-d69ab9ad1e10"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 0s 3ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[9.9677700e-01],\n",
              "       [2.2558276e-01],\n",
              "       [7.3814642e-04],\n",
              "       [2.3152212e-04],\n",
              "       [1.7952776e-01],\n",
              "       [9.9988663e-01],\n",
              "       [9.9979180e-01],\n",
              "       [2.7436859e-04],\n",
              "       [1.7627304e-04],\n",
              "       [9.9985206e-01]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert model 3 pred probs to labels\n",
        "model_3_preds = tf.squeeze(tf.round(model_3_pred_probs))\n",
        "model_3_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Ke_8Y2n4pRo",
        "outputId": "8d4db801-67fe-4607-f6cb-022b8bdb4178"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 0., 0., 0., 0., 1., 1., 0., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate model 3 results\n",
        "model_3_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_3_preds)\n",
        "model_3_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xw8UpPa55DxE",
        "outputId": "8d34f884-7c51-4284-9425-83fba6f91c46"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 73.0971128608924,\n",
              " 'precision': 0.7308822912966644,\n",
              " 'recall': 0.7309711286089239,\n",
              " 'f1': 0.7309254454828463}"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model 4: Bidirectional RNN\n",
        "\n",
        "Normal RNN's go from left to right (just like you'd read an English sentence) however, a bidirectional RNN goes from right to left as well as left to right."
      ],
      "metadata": {
        "id": "h7zOOUJj5deA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Build a bidirectional RNN in tensorflow\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=tf.string)\n",
        "x = text_vectorizer(inputs)\n",
        "x = embedding(x)\n",
        "#x = layers.Bidirectional(layers.LSTM(64, return_sequences=True))(x)\n",
        "x = layers.Bidirectional(layers.LSTM(64))(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_4 = tf.keras.Model(inputs, outputs, name=\"model_4_bidirectional\")"
      ],
      "metadata": {
        "id": "TloOZ_0w-L3D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a summary\n",
        "model_4.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T5YAYLzd_YPA",
        "outputId": "225d548f-3513-49eb-bf12-2689a89e8cee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_4_bidirectional\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_4 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " bidirectional (Bidirection  (None, 128)               98816     \n",
            " al)                                                             \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1378945 (5.26 MB)\n",
            "Trainable params: 1378945 (5.26 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile the model\n",
        "model_4.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=\"Adam\",\n",
        "                metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "5DjmEmm___qU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model\n",
        "model_4_history = model_4.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences,val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
        "                                                                     \"model_4_bidirectional\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZDxOXheANdq",
        "outputId": "065fbe49-258b-4230-e5d8-9094117b90fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/model_4_bidirectional/20240826-160324\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 13s 45ms/step - loss: 0.1058 - accuracy: 0.9731 - val_loss: 1.1247 - val_accuracy: 0.7310\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 3s 14ms/step - loss: 0.0536 - accuracy: 0.9775 - val_loss: 1.2930 - val_accuracy: 0.7310\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 2s 10ms/step - loss: 0.0434 - accuracy: 0.9806 - val_loss: 1.3458 - val_accuracy: 0.7323\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 2s 10ms/step - loss: 0.0453 - accuracy: 0.9796 - val_loss: 1.3129 - val_accuracy: 0.7205\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 3s 14ms/step - loss: 0.0401 - accuracy: 0.9822 - val_loss: 1.4681 - val_accuracy: 0.7310\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions with our bidirectional model\n",
        "model_4_pred_probs = model_4.predict(val_sentences)\n",
        "model_4_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rhYHc1-QA1eN",
        "outputId": "7579c26e-6e41-42bc-bf9f-f860ff0ec812"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 3ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[9.96121347e-01],\n",
              "       [2.75516063e-01],\n",
              "       [1.04445426e-04],\n",
              "       [5.98391744e-05],\n",
              "       [4.07717019e-01],\n",
              "       [9.99933004e-01],\n",
              "       [9.99935746e-01],\n",
              "       [8.77438069e-05],\n",
              "       [2.85973001e-05],\n",
              "       [9.99959707e-01]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert pred probs to pred labels\n",
        "model_4_preds = tf.squeeze(tf.round(model_4_pred_probs))\n",
        "model_4_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0nDW8SJVBcN5",
        "outputId": "41643d9b-acb4-4914-c864-8b404f8fae56"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 0., 0., 0., 0., 1., 1., 0., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate the results of our bidirectional model\n",
        "model_4_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_4_preds)\n",
        "model_4_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M7L_1GjrCHVm",
        "outputId": "84e97d62-d2f3-43b2-a428-aaf97c63b3c4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 73.0971128608924,\n",
              " 'precision': 0.7335539313817079,\n",
              " 'recall': 0.7309711286089239,\n",
              " 'f1': 0.7317080094292809}"
            ]
          },
          "metadata": {},
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Convolutional Neural Networks for Text (and other types of sequences)\n",
        "\n",
        "We've used CNNs for images but images are typically 2D (height x width)...however, our data is 1D.\n",
        "\n",
        "Previously we've used Conv2D for our image data , but now we're going to use Conv1D.\n",
        "\n",
        "The typical structure of a Conv1D model for sequences (in our case, text) looks like this:\n",
        "\n",
        "```\n",
        "Input (text) -> Tokenization -> Embedding -> Layer(s) (typically Conv1D + pooling) -> Outputs (class probabilities)\n",
        "```"
      ],
      "metadata": {
        "id": "R0VjV-c5C0QM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model 5: Conv1D"
      ],
      "metadata": {
        "id": "Fd7wPLyUFCkF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Test out our embedding layer, Conv1D layer asndf max pooling\n",
        "from tensorflow.keras import layers\n",
        "embedding_test = embedding(text_vectorizer([\"this is a test sentence\"]))\n",
        "conv_1d = layers.Conv1D(filters=32,\n",
        "                        kernel_size=5, # this is also referred to as an ngram of 5 (meaning it looks at 5 words at a time)\n",
        "                        strides=1, # default\n",
        "                        activation=\"relu\",\n",
        "                        padding=\"same\") # default = \"valid\", the output is smaller than the input shape, \"same\" means output is same as input\n",
        "conv_1d_output = conv_1d(embedding_test) # pass test embedding through conv1D layer\n",
        "max_pool = layers.GlobalMaxPool1D()\n",
        "max_pool_output = max_pool(conv_1d_output) # equivalent to \"get the most important feature\"\n",
        "embedding_test.shape, conv_1d_output.shape, max_pool_output.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FYpm0OscFYUZ",
        "outputId": "3b96e101-0a11-4799-81be-babd089b36e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([1, 15, 128]), TensorShape([1, 15, 32]), TensorShape([1, 32]))"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X0RIqUq1HB7T",
        "outputId": "6b19c20a-8f6c-4d95-a014-3bf8b1a4739e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15, 128), dtype=float32, numpy=\n",
              "array([[[ 0.04618527, -0.0701745 , -0.07419334, ...,  0.02595809,\n",
              "          0.0332571 ,  0.02125148],\n",
              "        [ 0.03886741, -0.02187036,  0.01731213, ..., -0.00366324,\n",
              "          0.01545165,  0.0117384 ],\n",
              "        [ 0.0357705 , -0.04654601, -0.03233727, ...,  0.03305587,\n",
              "          0.02473041,  0.05499237],\n",
              "        ...,\n",
              "        [-0.01742523, -0.00262937, -0.00753612, ...,  0.02129435,\n",
              "          0.02411881,  0.02747238],\n",
              "        [-0.01742523, -0.00262937, -0.00753612, ...,  0.02129435,\n",
              "          0.02411881,  0.02747238],\n",
              "        [-0.01742523, -0.00262937, -0.00753612, ...,  0.02129435,\n",
              "          0.02411881,  0.02747238]]], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "conv_1d_output"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iqwM3kY2JmD4",
        "outputId": "6a8134e6-6e42-46c8-b419-2ec36d90b401"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15, 32), dtype=float32, numpy=\n",
              "array([[[0.        , 0.07005733, 0.01847008, 0.        , 0.        ,\n",
              "         0.        , 0.        , 0.01301908, 0.        , 0.03740363,\n",
              "         0.        , 0.02681712, 0.04972615, 0.01540109, 0.03885901,\n",
              "         0.        , 0.018817  , 0.00199707, 0.00718727, 0.02438145,\n",
              "         0.02730734, 0.06497042, 0.03106698, 0.        , 0.02048746,\n",
              "         0.        , 0.        , 0.02492594, 0.        , 0.        ,\n",
              "         0.        , 0.01573884],\n",
              "        [0.        , 0.        , 0.00564125, 0.        , 0.03863302,\n",
              "         0.        , 0.0087622 , 0.00336095, 0.        , 0.01231745,\n",
              "         0.0043183 , 0.        , 0.08756079, 0.        , 0.03681967,\n",
              "         0.00181954, 0.05325651, 0.        , 0.        , 0.01064526,\n",
              "         0.00838085, 0.00469184, 0.        , 0.04460097, 0.        ,\n",
              "         0.        , 0.        , 0.        , 0.        , 0.07669897,\n",
              "         0.        , 0.05586149],\n",
              "        [0.        , 0.00196721, 0.04055609, 0.08487642, 0.        ,\n",
              "         0.06496571, 0.02125745, 0.026429  , 0.02532787, 0.03947884,\n",
              "         0.        , 0.04100617, 0.        , 0.02743264, 0.01255975,\n",
              "         0.03096624, 0.04957233, 0.01588907, 0.01347842, 0.        ,\n",
              "         0.00768194, 0.        , 0.        , 0.10842311, 0.03231592,\n",
              "         0.        , 0.        , 0.        , 0.01069386, 0.        ,\n",
              "         0.02901004, 0.        ],\n",
              "        [0.        , 0.07007068, 0.09290911, 0.07793051, 0.        ,\n",
              "         0.        , 0.04332832, 0.0331891 , 0.        , 0.00845364,\n",
              "         0.02869822, 0.        , 0.        , 0.        , 0.        ,\n",
              "         0.03871432, 0.03801163, 0.        , 0.        , 0.        ,\n",
              "         0.04606551, 0.        , 0.        , 0.        , 0.        ,\n",
              "         0.        , 0.01310195, 0.        , 0.        , 0.02108041,\n",
              "         0.        , 0.        ],\n",
              "        [0.        , 0.        , 0.02249213, 0.07625496, 0.02121208,\n",
              "         0.        , 0.        , 0.02085886, 0.04685993, 0.01190009,\n",
              "         0.        , 0.02224639, 0.08138309, 0.03377869, 0.        ,\n",
              "         0.01726107, 0.01962958, 0.05332889, 0.00587157, 0.02508447,\n",
              "         0.        , 0.02788005, 0.        , 0.08083268, 0.        ,\n",
              "         0.03117438, 0.        , 0.        , 0.        , 0.01565875,\n",
              "         0.        , 0.02838142],\n",
              "        [0.        , 0.03051517, 0.        , 0.05206681, 0.04228619,\n",
              "         0.        , 0.        , 0.00471833, 0.03718115, 0.00618146,\n",
              "         0.        , 0.        , 0.        , 0.06499131, 0.        ,\n",
              "         0.        , 0.03007934, 0.        , 0.05056716, 0.06528497,\n",
              "         0.        , 0.        , 0.        , 0.09209727, 0.        ,\n",
              "         0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "         0.        , 0.06633796],\n",
              "        [0.        , 0.        , 0.03345515, 0.        , 0.06120649,\n",
              "         0.        , 0.        , 0.        , 0.03173628, 0.03297503,\n",
              "         0.02456879, 0.02387407, 0.        , 0.02787571, 0.        ,\n",
              "         0.01667142, 0.02479795, 0.        , 0.0335567 , 0.03588133,\n",
              "         0.02037942, 0.08381011, 0.00409054, 0.03254517, 0.01425291,\n",
              "         0.        , 0.01265037, 0.00212835, 0.0129319 , 0.        ,\n",
              "         0.        , 0.02023955],\n",
              "        [0.        , 0.01449866, 0.02391999, 0.        , 0.05521786,\n",
              "         0.        , 0.01536021, 0.        , 0.03678311, 0.0012249 ,\n",
              "         0.01756932, 0.        , 0.02667109, 0.02699533, 0.        ,\n",
              "         0.03290374, 0.04666434, 0.        , 0.02183346, 0.03189652,\n",
              "         0.00696546, 0.03302959, 0.02549139, 0.01327961, 0.        ,\n",
              "         0.        , 0.        , 0.00979578, 0.01812546, 0.02100895,\n",
              "         0.        , 0.02581034],\n",
              "        [0.        , 0.01449867, 0.02391999, 0.        , 0.05521786,\n",
              "         0.        , 0.01536021, 0.        , 0.0367831 , 0.0012249 ,\n",
              "         0.01756932, 0.        , 0.0266711 , 0.02699533, 0.        ,\n",
              "         0.03290374, 0.04666434, 0.        , 0.02183345, 0.03189653,\n",
              "         0.00696546, 0.03302961, 0.0254914 , 0.0132796 , 0.        ,\n",
              "         0.        , 0.        , 0.00979578, 0.01812546, 0.02100896,\n",
              "         0.        , 0.02581035],\n",
              "        [0.        , 0.01449866, 0.02391998, 0.        , 0.05521786,\n",
              "         0.        , 0.01536021, 0.        , 0.0367831 , 0.0012249 ,\n",
              "         0.01756932, 0.        , 0.0266711 , 0.02699533, 0.        ,\n",
              "         0.03290373, 0.04666434, 0.        , 0.02183345, 0.03189653,\n",
              "         0.00696546, 0.0330296 , 0.02549139, 0.01327961, 0.        ,\n",
              "         0.        , 0.        , 0.00979578, 0.01812545, 0.02100896,\n",
              "         0.        , 0.02581035],\n",
              "        [0.        , 0.01449867, 0.02391999, 0.        , 0.05521785,\n",
              "         0.        , 0.01536021, 0.        , 0.0367831 , 0.00122491,\n",
              "         0.01756932, 0.        , 0.0266711 , 0.02699533, 0.        ,\n",
              "         0.03290374, 0.04666434, 0.        , 0.02183346, 0.03189652,\n",
              "         0.00696546, 0.0330296 , 0.02549139, 0.01327961, 0.        ,\n",
              "         0.        , 0.        , 0.00979578, 0.01812546, 0.02100895,\n",
              "         0.        , 0.02581034],\n",
              "        [0.        , 0.01449867, 0.02391999, 0.        , 0.05521785,\n",
              "         0.        , 0.01536022, 0.        , 0.0367831 , 0.00122491,\n",
              "         0.01756933, 0.        , 0.0266711 , 0.02699533, 0.        ,\n",
              "         0.03290374, 0.04666434, 0.        , 0.02183345, 0.03189652,\n",
              "         0.00696546, 0.0330296 , 0.02549139, 0.0132796 , 0.        ,\n",
              "         0.        , 0.        , 0.00979578, 0.01812546, 0.02100896,\n",
              "         0.        , 0.02581033],\n",
              "        [0.        , 0.01449867, 0.02391998, 0.        , 0.05521786,\n",
              "         0.        , 0.01536021, 0.        , 0.03678311, 0.0012249 ,\n",
              "         0.01756932, 0.        , 0.0266711 , 0.02699533, 0.        ,\n",
              "         0.03290374, 0.04666434, 0.        , 0.02183345, 0.03189653,\n",
              "         0.00696546, 0.0330296 , 0.02549139, 0.0132796 , 0.        ,\n",
              "         0.        , 0.        , 0.00979578, 0.01812546, 0.02100896,\n",
              "         0.        , 0.02581035],\n",
              "        [0.        , 0.0123353 , 0.01146854, 0.        , 0.03880977,\n",
              "         0.        , 0.02738417, 0.        , 0.03943735, 0.00014639,\n",
              "         0.00592501, 0.        , 0.01417978, 0.01105053, 0.00753304,\n",
              "         0.0173788 , 0.04124837, 0.        , 0.00242351, 0.02205322,\n",
              "         0.00163374, 0.02873465, 0.0213966 , 0.00696217, 0.        ,\n",
              "         0.        , 0.        , 0.01047917, 0.01666064, 0.02850589,\n",
              "         0.        , 0.01093733],\n",
              "        [0.        , 0.00417159, 0.00983999, 0.        , 0.03062839,\n",
              "         0.        , 0.03365763, 0.        , 0.01641428, 0.01963338,\n",
              "         0.01703279, 0.        , 0.01543061, 0.01276832, 0.00735481,\n",
              "         0.01156158, 0.02605144, 0.        , 0.0046473 , 0.        ,\n",
              "         0.00568774, 0.01156658, 0.02570749, 0.        , 0.        ,\n",
              "         0.        , 0.        , 0.00937787, 0.02273368, 0.03072735,\n",
              "         0.        , 0.        ]]], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max_pool_output"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cJAeirN3JsSJ",
        "outputId": "0519d899-7cdd-409d-ca1b-82c80c8c908c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 32), dtype=float32, numpy=\n",
              "array([[0.        , 0.07007068, 0.09290911, 0.08487642, 0.06120649,\n",
              "        0.06496571, 0.04332832, 0.0331891 , 0.04685993, 0.03947884,\n",
              "        0.02869822, 0.04100617, 0.08756079, 0.06499131, 0.03885901,\n",
              "        0.03871432, 0.05325651, 0.05332889, 0.05056716, 0.06528497,\n",
              "        0.04606551, 0.08381011, 0.03106698, 0.10842311, 0.03231592,\n",
              "        0.03117438, 0.01310195, 0.02492594, 0.02273368, 0.07669897,\n",
              "        0.02901004, 0.06633796]], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create 1-dimensional convolutional layer to model sequences\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
        "x = text_vectorizer(inputs)\n",
        "x = embedding(x)\n",
        "x = layers.Conv1D(filters=64, kernel_size=5, strides=1, activation=\"relu\", padding=\"valid\")(x)\n",
        "x = layers.GlobalMaxPool1D()(x)\n",
        "#x = layers.Dense(64, activation=\"relu\")(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_5 = tf.keras.Model(inputs, outputs, name=\"model_5_Conv1D\")\n",
        "\n",
        "# Compile Conv1D\n",
        "model_5.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "# Get a summary of our Conv1D model\n",
        "model_5.summary()"
      ],
      "metadata": {
        "id": "9Y5L73PXJ4Sq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b562b20e-f6f2-4d46-f11e-1b4c4f8d0cc0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_5_Conv1D\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_5 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding (Embedding)       (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " conv1d_1 (Conv1D)           (None, 11, 64)            41024     \n",
            "                                                                 \n",
            " global_max_pooling1d_1 (Gl  (None, 64)                0         \n",
            " obalMaxPooling1D)                                               \n",
            "                                                                 \n",
            " dense_4 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1321089 (5.04 MB)\n",
            "Trainable params: 1321089 (5.04 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model\n",
        "model_5_history = model_5.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences,val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
        "                                                                     \"Conv1D\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PJvhgV3Rv5rJ",
        "outputId": "aecd5b12-33e8-4e7c-ae64-a5cd782f4eb2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/Conv1D/20240826-160349\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 9s 34ms/step - loss: 0.1181 - accuracy: 0.9623 - val_loss: 1.0185 - val_accuracy: 0.7310\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 3s 14ms/step - loss: 0.0732 - accuracy: 0.9714 - val_loss: 1.1899 - val_accuracy: 0.7349\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.0595 - accuracy: 0.9769 - val_loss: 1.2885 - val_accuracy: 0.7283\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 1s 7ms/step - loss: 0.0544 - accuracy: 0.9772 - val_loss: 1.3460 - val_accuracy: 0.7283\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 2s 7ms/step - loss: 0.0515 - accuracy: 0.9791 - val_loss: 1.4019 - val_accuracy: 0.7192\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make some predictions with our Conv1D model\n",
        "model_5_pred_probs = model_5.predict(val_sentences)\n",
        "model_5_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kV3nYk8dwdoS",
        "outputId": "83cbb87b-f645-437a-e3e8-c5dc836d542d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 0s 4ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[9.9915147e-01],\n",
              "       [9.9988949e-01],\n",
              "       [2.4530600e-06],\n",
              "       [1.3084286e-06],\n",
              "       [1.7545819e-01],\n",
              "       [9.9999988e-01],\n",
              "       [1.0000000e+00],\n",
              "       [1.3574862e-05],\n",
              "       [3.0979718e-05],\n",
              "       [9.9999988e-01]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert model 5 pred probs to labels\n",
        "model_5_preds = tf.squeeze(tf.round(model_5_pred_probs))\n",
        "model_5_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vslfpzhtwxy6",
        "outputId": "ce06b265-f426-4be5-c4ff-19a6d5d80f7a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 1., 0., 0., 0., 1., 1., 0., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluate model 5 predictions\n",
        "model_5_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_5_preds)\n",
        "model_5_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D-bo_tIlw_cx",
        "outputId": "d272d0e7-2a48-422f-c385-042ceb4cf29d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 71.91601049868767,\n",
              " 'precision': 0.7222708272577039,\n",
              " 'recall': 0.7191601049868767,\n",
              " 'f1': 0.7200034152753113}"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "baseline_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YPliKiOIxS70",
        "outputId": "49556b4f-2e71-44aa-dd1b-b298b27ea900"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 79.79002624671917,\n",
              " 'precision': 0.8049118114708822,\n",
              " 'recall': 0.7979002624671916,\n",
              " 'f1': 0.7932923017133588}"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model 6: TensorFlow Hub Pretrained Sentence Encoder\n",
        "\n",
        "Now we've built a few of our own models, let's try and use transfer learning for NLP, specifically using TensorFlow Hub's Universal Sentence Encoder."
      ],
      "metadata": {
        "id": "aFmVrUKMxfQp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sample_sentence"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "disnZ_u11DHL",
        "outputId": "520c013e-7e85-4f0c-ca46-b2570298b6bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"There's a flood in my street\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow_hub as hub\n",
        "embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\")\n",
        "embed_samples = embed([sample_sentence, \"When you can the universal sentence encoder on a sentence, it turns it into numbers.\"])\n",
        "print(embed_samples[0][:50])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JXZRLiCC0Mlc",
        "outputId": "08d4b3b6-374f-44de-f87b-046c38636435"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(\n",
            "[-0.01157028  0.0248591   0.02878048 -0.012715    0.03971538  0.0882776\n",
            "  0.02680984  0.05589836 -0.0106873  -0.00597291  0.00639323 -0.01819518\n",
            "  0.00030813  0.09105888  0.05874644 -0.03180628  0.01512474 -0.05162929\n",
            "  0.00991367 -0.06865347 -0.04209306  0.02678981  0.03011006  0.00321069\n",
            " -0.00337973 -0.04787357  0.0226672  -0.00985925 -0.04063613 -0.01292092\n",
            " -0.04666384  0.05630299 -0.03949255  0.00517686  0.02495829 -0.0701444\n",
            "  0.02871508  0.04947684 -0.00633979 -0.08960192  0.02807118 -0.00808364\n",
            " -0.01360602  0.0599865  -0.10361787 -0.05195374  0.00232954 -0.02332531\n",
            " -0.03758105  0.03327728], shape=(50,), dtype=float32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### The **USE** (Universal Sentence Encoder) takes an english sentence of variable length as input and turns it into a feature vector of size 512 as output , based on the patterns it had previously trained on."
      ],
      "metadata": {
        "id": "tp5Nta8k0jTX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a Keras layer using the USE pretrained layer from TensorFlow Hub\n",
        "sentence_encoder_layer = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder/4\",\n",
        "                                        input_shape=[], # shape of inputs coming to our model,\n",
        "                                        dtype=tf.string, # data type of inputs\n",
        "                                        trainable=False, # keep the pretrained weights (we'll create a feature extractor)\n",
        "                                        name=\"USE\")"
      ],
      "metadata": {
        "id": "IY6OZt9t2lxX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a model using the Sequential API\n",
        "model_6 = tf.keras.Sequential([\n",
        "  sentence_encoder_layer, # take in sentences and then encode them into an embedding\n",
        "  layers.Dense(64, activation=\"relu\"),\n",
        "  layers.Dense(1, activation=\"sigmoid\")],\n",
        "  name=\"model_6_USE\")\n",
        "\n",
        "# Compile the model\n",
        "model_6.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "model_6.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4H-7eUdl7fEG",
        "outputId": "6212eb54-11fe-48ef-b641-4e6f70c184cf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_6_USE\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " USE (KerasLayer)            (None, 512)               256797824 \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 256830721 (979.73 MB)\n",
            "Trainable params: 32897 (128.50 KB)\n",
            "Non-trainable params: 256797824 (979.61 MB)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Train a classifier on top of USE pretrained embeddings\n",
        "model_6_history = model_6.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR, \"tf_hub_sentence_encoder\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8AtknwVH7_zW",
        "outputId": "bbe23815-57e7-4875-c831-8a14c83d6d54"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/tf_hub_sentence_encoder/20240826-160444\n",
            "Epoch 1/5\n",
            "215/215 [==============================] - 7s 19ms/step - loss: 0.4985 - accuracy: 0.7879 - val_loss: 0.4506 - val_accuracy: 0.7808\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 3s 13ms/step - loss: 0.4140 - accuracy: 0.8178 - val_loss: 0.4372 - val_accuracy: 0.8005\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 3s 13ms/step - loss: 0.4000 - accuracy: 0.8254 - val_loss: 0.4327 - val_accuracy: 0.8031\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 3s 13ms/step - loss: 0.3912 - accuracy: 0.8281 - val_loss: 0.4335 - val_accuracy: 0.8018\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 4s 17ms/step - loss: 0.3849 - accuracy: 0.8313 - val_loss: 0.4296 - val_accuracy: 0.8058\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions with USE TF Hub model\n",
        "model_6_pred_probs = model_6.predict(val_sentences)\n",
        "model_6_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zewGgBTv8VZt",
        "outputId": "629cf230-93d0-4ad8-f9ec-7e2cabc80828"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 9ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.8668142 ],\n",
              "       [0.9075005 ],\n",
              "       [0.0973869 ],\n",
              "       [0.07067621],\n",
              "       [0.20556235],\n",
              "       [0.61094177],\n",
              "       [0.9839909 ],\n",
              "       [0.07736097],\n",
              "       [0.12509228],\n",
              "       [0.98290455]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert prediction probabilities into labels\n",
        "model_6_preds = tf.squeeze(tf.round(model_6_pred_probs))\n",
        "model_6_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gRkljFdV8gpd",
        "outputId": "19f3e659-ecd2-48b4-dff0-7b774e9c1f27"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 1., 0., 0., 0., 1., 1., 0., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate model 6 performance metrics\n",
        "model_6_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_6_preds)\n",
        "model_6_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J_WFhiz_8qo0",
        "outputId": "638504d5-1999-40ec-c87b-0d8f785b0f45"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 80.5774278215223,\n",
              " 'precision': 0.8057420069627774,\n",
              " 'recall': 0.8057742782152231,\n",
              " 'f1': 0.8044743227851235}"
            ]
          },
          "metadata": {},
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "baseline_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QatdGr0w8x8E",
        "outputId": "5e429b5b-fbd6-44de-e15d-883ddbb2bc50"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 79.79002624671917,\n",
              " 'precision': 0.8049118114708822,\n",
              " 'recall': 0.7979002624671916,\n",
              " 'f1': 0.7932923017133588}"
            ]
          },
          "metadata": {},
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model 7 : TF Hub pretrained USE but ith only 10% of training data\n",
        "\n",
        "Transfer Learning really helps when you don't have a large dataset.\n",
        "\n",
        "To see how our model performs on a smaller dataset, let's replicate `model_6` except we'll train it on 10% of the data."
      ],
      "metadata": {
        "id": "4ipv0_Y_9I6b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "  # ## NOTE: Making data splits like below leads to data leakage (model_7 trained on 10% data, outperforms model_6 trained on 100% of the data)\n",
        "  # ## DO NOT MAKE DATA SPLITS WHICH LEAK DATA FROM VALIDATION/TEST SETS INTO TRAINING SET\n",
        "\n",
        "  # # Create subsets of 10% of the training data\n",
        "  # train_10_percent = train_df_shuffled[[\"text\", \"target\"]].sample(frac=0.1, random_state=69)\n",
        "  # #train_10_percent.head(), len(train_10_percent)\n",
        "  # train_sentences_10_percent = train_10_percent[\"text\"].to_list()\n",
        "  # train_labels_10_percent = train_10_percent[\"target\"].to_list()\n",
        "  # len(train_sentences_10_percent), len(train_labels_10_percent)"
      ],
      "metadata": {
        "id": "jLr9Ap7x-to7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "🔑**Note:** Be *very* careful when creating training/val/test splits that you don't leak data across the datasets, otherwise your model evaluation metrics will be wrong. If something looks too good to be true ( a model trained on 10% of the data, outperforming the model trained on 100% of the data) trust your gut and go back through to find where the error may lie."
      ],
      "metadata": {
        "id": "6WabxgdnIqyZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Making a better data split\n",
        "train_10_percent_split = int(0.1 * len(train_sentences))\n",
        "train_sentences_10_percent = train_sentences[:train_10_percent_split]\n",
        "train_labels_10_percent = train_labels[:train_10_percent_split]"
      ],
      "metadata": {
        "id": "bfkeO77kGYf-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the number of each label in the updated data subset\n",
        "pd.Series(np.array(train_labels_10_percent)).value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 147
        },
        "id": "a_2B7aqh_ZGz",
        "outputId": "d8f09945-938f-4d07-a0de-10457a00b6f2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    394\n",
              "1    291\n",
              "Name: count, dtype: int64"
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>394</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>291</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div><br><label><b>dtype:</b> int64</label>"
            ]
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "To recreate a model the same as a previous model you've created you can use the `tf.keras.models.clone_model()` method, see more here: https://www.tensorflow.org/api_docs/python/tf/keras/models/clone_model"
      ],
      "metadata": {
        "id": "LLhcQ5zhAHKz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's build a model same as model 6\n",
        "model_7 = tf.keras.models.clone_model(model_6)\n",
        "\n",
        "# Compile the model\n",
        "model_7.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "# Get a summary\n",
        "model_7.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wL3k4KLF_iRb",
        "outputId": "281a62e2-49a8-4e51-e34a-85ffa66bfa51"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_6_USE\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " USE (KerasLayer)            (None, 512)               256797824 \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 256830721 (979.73 MB)\n",
            "Trainable params: 32897 (128.50 KB)\n",
            "Non-trainable params: 256797824 (979.61 MB)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model to the 10% training data subset\n",
        "model_7_history = model_7.fit(train_sentences_10_percent,\n",
        "                               train_labels_10_percent,\n",
        "                               epochs=5,\n",
        "                               validation_data=(val_sentences, val_labels),\n",
        "                               callbacks=[create_tensorboard_callback(SAVE_DIR, \"tf_hub_sentence_encoder_10_percent_correct_split\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zy3nwD3eA5c5",
        "outputId": "a667a74c-d69c-42d2-c060-98fd57dc7211"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/tf_hub_sentence_encoder_10_percent_correct_split/20240826-160514\n",
            "Epoch 1/5\n",
            "22/22 [==============================] - 4s 49ms/step - loss: 0.6787 - accuracy: 0.6394 - val_loss: 0.6542 - val_accuracy: 0.7507\n",
            "Epoch 2/5\n",
            "22/22 [==============================] - 1s 27ms/step - loss: 0.6183 - accuracy: 0.8321 - val_loss: 0.5964 - val_accuracy: 0.7703\n",
            "Epoch 3/5\n",
            "22/22 [==============================] - 0s 21ms/step - loss: 0.5425 - accuracy: 0.8394 - val_loss: 0.5362 - val_accuracy: 0.7638\n",
            "Epoch 4/5\n",
            "22/22 [==============================] - 0s 23ms/step - loss: 0.4735 - accuracy: 0.8350 - val_loss: 0.4940 - val_accuracy: 0.7651\n",
            "Epoch 5/5\n",
            "22/22 [==============================] - 0s 22ms/step - loss: 0.4232 - accuracy: 0.8482 - val_loss: 0.4736 - val_accuracy: 0.7795\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions with the model trained on 10% of the data\n",
        "model_7_pred_probs = model_7.predict(val_sentences)\n",
        "model_7_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xAbE5jlhBVD6",
        "outputId": "e1ebf3f6-7ddc-4f41-cf55-297d16fcfdf4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 9ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.8217927 ],\n",
              "       [0.7313086 ],\n",
              "       [0.5270055 ],\n",
              "       [0.11068874],\n",
              "       [0.3392835 ],\n",
              "       [0.70316917],\n",
              "       [0.9084253 ],\n",
              "       [0.38323417],\n",
              "       [0.19403408],\n",
              "       [0.91485184]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Turn pred probs into labels\n",
        "model_7_preds = tf.squeeze(tf.round(model_7_pred_probs))\n",
        "model_7_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LlxRnA7jBgVa",
        "outputId": "803a1fe9-5b89-4b04-803b-0cafa86d696d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 1., 1., 0., 0., 1., 1., 0., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluate model_7 predictions\n",
        "model_7_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_7_preds)\n",
        "model_7_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LeO-xPW5BnGL",
        "outputId": "30362506-c452-40d9-90ef-534ac179f8b0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 77.95275590551181,\n",
              " 'precision': 0.7798543396423501,\n",
              " 'recall': 0.7795275590551181,\n",
              " 'f1': 0.7796693568429955}"
            ]
          },
          "metadata": {},
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Comparing the performance of each of our models"
      ],
      "metadata": {
        "id": "clWvN-_fBt0y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Combine model results into a dataframe\n",
        "all_model_results = pd.DataFrame({\"0_baseline\": baseline_results,\n",
        "                                 \"1_simple_dense\": model_1_results,\n",
        "                                 \"2_lstm\": model_2_results,\n",
        "                                 \"3_gru\": model_3_results,\n",
        "                                 \"4_bidirectional\": model_4_results,\n",
        "                                 \"5_conv1d\": model_5_results,\n",
        "                                 \"6_tf_hub_use_encoder\": model_6_results,\n",
        "                                 \"7_tf_hub_use_encoder_10_percent\": model_7_results})\n",
        "all_model_results = all_model_results.transpose()\n",
        "all_model_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "tIv5zDczJ9Ur",
        "outputId": "d94fa07b-303a-4a72-d867-c4299b41e6c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                  accuracy  precision    recall        f1\n",
              "0_baseline                       79.790026   0.804912  0.797900  0.793292\n",
              "1_simple_dense                   78.346457   0.783490  0.783465  0.781572\n",
              "2_lstm                           73.622047   0.735678  0.736220  0.735887\n",
              "3_gru                            73.097113   0.730882  0.730971  0.730925\n",
              "4_bidirectional                  73.097113   0.733554  0.730971  0.731708\n",
              "5_conv1d                         71.916010   0.722271  0.719160  0.720003\n",
              "6_tf_hub_use_encoder             80.577428   0.805742  0.805774  0.804474\n",
              "7_tf_hub_use_encoder_10_percent  77.952756   0.779854  0.779528  0.779669"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-feb1fd4e-154b-4c86-aee3-bade79e9c133\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0_baseline</th>\n",
              "      <td>79.790026</td>\n",
              "      <td>0.804912</td>\n",
              "      <td>0.797900</td>\n",
              "      <td>0.793292</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1_simple_dense</th>\n",
              "      <td>78.346457</td>\n",
              "      <td>0.783490</td>\n",
              "      <td>0.783465</td>\n",
              "      <td>0.781572</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2_lstm</th>\n",
              "      <td>73.622047</td>\n",
              "      <td>0.735678</td>\n",
              "      <td>0.736220</td>\n",
              "      <td>0.735887</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3_gru</th>\n",
              "      <td>73.097113</td>\n",
              "      <td>0.730882</td>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.730925</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4_bidirectional</th>\n",
              "      <td>73.097113</td>\n",
              "      <td>0.733554</td>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.731708</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5_conv1d</th>\n",
              "      <td>71.916010</td>\n",
              "      <td>0.722271</td>\n",
              "      <td>0.719160</td>\n",
              "      <td>0.720003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6_tf_hub_use_encoder</th>\n",
              "      <td>80.577428</td>\n",
              "      <td>0.805742</td>\n",
              "      <td>0.805774</td>\n",
              "      <td>0.804474</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7_tf_hub_use_encoder_10_percent</th>\n",
              "      <td>77.952756</td>\n",
              "      <td>0.779854</td>\n",
              "      <td>0.779528</td>\n",
              "      <td>0.779669</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-feb1fd4e-154b-4c86-aee3-bade79e9c133')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-feb1fd4e-154b-4c86-aee3-bade79e9c133 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-feb1fd4e-154b-4c86-aee3-bade79e9c133');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-f3ec3bf2-70c7-423b-8d46-87c213c79981\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f3ec3bf2-70c7-423b-8d46-87c213c79981')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-f3ec3bf2-70c7-423b-8d46-87c213c79981 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_a7bf7d92-b53a-4f8e-8ee2-9a3bf5b1f92d\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('all_model_results')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_a7bf7d92-b53a-4f8e-8ee2-9a3bf5b1f92d button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('all_model_results');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "all_model_results",
              "summary": "{\n  \"name\": \"all_model_results\",\n  \"rows\": 8,\n  \"fields\": [\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.4600475005066276,\n        \"min\": 71.91601049868767,\n        \"max\": 80.5774278215223,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          79.79002624671917,\n          78.34645669291339,\n          80.5774278215223\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"precision\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.035018544432441207,\n        \"min\": 0.7222708272577039,\n        \"max\": 0.8057420069627774,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7834899124733871,\n          0.7222708272577039,\n          0.8049118114708822\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"recall\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.034600475005066286,\n        \"min\": 0.7191601049868767,\n        \"max\": 0.8057742782152231,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          0.7979002624671916,\n          0.7834645669291339,\n          0.8057742782152231\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"f1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.033305389892773984,\n        \"min\": 0.7200034152753113,\n        \"max\": 0.8044743227851235,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7815717498014031,\n          0.7200034152753113,\n          0.7932923017133588\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Reduce the accuracy to the same scale as the other metrics\n",
        "all_model_results[\"accuracy\"] = all_model_results[\"accuracy\"]/100\n",
        "all_model_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "ouviJFtoLCRQ",
        "outputId": "45fc2cbf-1dc2-433e-e8af-62393ea2da99"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                 accuracy  precision    recall        f1\n",
              "0_baseline                       0.797900   0.804912  0.797900  0.793292\n",
              "1_simple_dense                   0.783465   0.783490  0.783465  0.781572\n",
              "2_lstm                           0.736220   0.735678  0.736220  0.735887\n",
              "3_gru                            0.730971   0.730882  0.730971  0.730925\n",
              "4_bidirectional                  0.730971   0.733554  0.730971  0.731708\n",
              "5_conv1d                         0.719160   0.722271  0.719160  0.720003\n",
              "6_tf_hub_use_encoder             0.805774   0.805742  0.805774  0.804474\n",
              "7_tf_hub_use_encoder_10_percent  0.779528   0.779854  0.779528  0.779669"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-00b0cb87-8b9d-477d-814a-e5278120697b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0_baseline</th>\n",
              "      <td>0.797900</td>\n",
              "      <td>0.804912</td>\n",
              "      <td>0.797900</td>\n",
              "      <td>0.793292</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1_simple_dense</th>\n",
              "      <td>0.783465</td>\n",
              "      <td>0.783490</td>\n",
              "      <td>0.783465</td>\n",
              "      <td>0.781572</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2_lstm</th>\n",
              "      <td>0.736220</td>\n",
              "      <td>0.735678</td>\n",
              "      <td>0.736220</td>\n",
              "      <td>0.735887</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3_gru</th>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.730882</td>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.730925</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4_bidirectional</th>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.733554</td>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.731708</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5_conv1d</th>\n",
              "      <td>0.719160</td>\n",
              "      <td>0.722271</td>\n",
              "      <td>0.719160</td>\n",
              "      <td>0.720003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6_tf_hub_use_encoder</th>\n",
              "      <td>0.805774</td>\n",
              "      <td>0.805742</td>\n",
              "      <td>0.805774</td>\n",
              "      <td>0.804474</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7_tf_hub_use_encoder_10_percent</th>\n",
              "      <td>0.779528</td>\n",
              "      <td>0.779854</td>\n",
              "      <td>0.779528</td>\n",
              "      <td>0.779669</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-00b0cb87-8b9d-477d-814a-e5278120697b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-00b0cb87-8b9d-477d-814a-e5278120697b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-00b0cb87-8b9d-477d-814a-e5278120697b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-f07a39b7-7baa-405a-b236-aa5ab6ead80f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f07a39b7-7baa-405a-b236-aa5ab6ead80f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-f07a39b7-7baa-405a-b236-aa5ab6ead80f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_f5860eb9-e613-4691-b9b9-599ac87b9951\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('all_model_results')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_f5860eb9-e613-4691-b9b9-599ac87b9951 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('all_model_results');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "all_model_results",
              "summary": "{\n  \"name\": \"all_model_results\",\n  \"rows\": 8,\n  \"fields\": [\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.03460047500506627,\n        \"min\": 0.7191601049868767,\n        \"max\": 0.8057742782152231,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          0.7979002624671917,\n          0.7834645669291339,\n          0.8057742782152231\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"precision\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.035018544432441207,\n        \"min\": 0.7222708272577039,\n        \"max\": 0.8057420069627774,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7834899124733871,\n          0.7222708272577039,\n          0.8049118114708822\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"recall\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.034600475005066286,\n        \"min\": 0.7191601049868767,\n        \"max\": 0.8057742782152231,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          0.7979002624671916,\n          0.7834645669291339,\n          0.8057742782152231\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"f1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.033305389892773984,\n        \"min\": 0.7200034152753113,\n        \"max\": 0.8044743227851235,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7815717498014031,\n          0.7200034152753113,\n          0.7932923017133588\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Plot and compare all of the model results\n",
        "all_model_results.plot(kind=\"bar\", figsize=(10,7)).legend(bbox_to_anchor=(1.0, 1.0));"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 825
        },
        "id": "vJEc3pYQMBsO",
        "outputId": "0e4cdf36-f6e1-4508-e240-58fc3fca69a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7IAAAMoCAYAAADyfdzRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB6A0lEQVR4nOzde3zO9eP/8ee1sc1mB8c5NOYUljEscopYhI9DfMupnKKoiYZQjpXDh3KKKIfoICSpPnxQi2QmGhvlPIdRNmca2dj2+6Nf16erjVyza2/v63rcb7frdtv1fr+u63pul8Oe1/v1fr0tWVlZWQIAAAAAwCTcjA4AAAAAAIA9KLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwlQJGB7gTmZmZ+vXXX+Xr6yuLxWJ0HAAAAAAGycrK0m+//aYyZcrIzY3jcq7KFEX2119/VVBQkNExAAAAANwjTp48qfvuu8/oGDCIKYqsr6+vpD/+sPr5+RmcBgAAAIBRrly5oqCgIGtHgGsyRZH9czqxn58fRRYAAAAApxy6OCaVAwAAAABMhSILAAAAADAViiwAAAAAwFRMcY4sAAAAANypzMxMpaenGx0DdipYsKDc3d3vaCxFFgAAAIDTSE9P17Fjx5SZmWl0FORCQECASpUq9Y+LeVFkAQAAADiFrKwsnT59Wu7u7goKCpKbG2dSmkVWVpauXbumM2fOSJJKly592/EUWQAAAABO4ebNm7p27ZrKlCkjb29vo+PAToUKFZIknTlzRiVLlrztNGM+ogAAAADgFDIyMiRJHh4eBidBbv35AcSNGzduO44iCwAAAMCp/NP5lbh33el7R5EFAAAAAJgKRRYAAAAAYCos9gQAAADAqQWPXJuvr3d8Stt8fT1HuHHjhgoWLGh0jFviiCwAAAAAGGz9+vVq3LixAgICVKxYMf3rX/9SYmKidf+pU6fUrVs3FS1aVD4+PgoPD9cPP/xg3f/VV1/pwQcflJeXl4oXL67HH3/cus9isWjNmjU2rxcQEKAlS5ZIko4fPy6LxaIVK1aoadOm8vLy0scff6zz58+rW7duKlu2rLy9vRUaGqpPPvnE5nkyMzM1depUVa5cWZ6enipXrpwmTpwoSWrevLkiIyNtxp89e1YeHh6Kjo6+q58XRRYAAAAADHb16lVFRUXpxx9/VHR0tNzc3PT4448rMzNTqampatq0qX755Rd9+eWXSkhI0Msvv6zMzExJ0tq1a/X444+rTZs22r17t6Kjo1WvXj27M4wcOVKDBw/W/v371apVK12/fl1169bV2rVr9dNPP+nZZ5/V008/rR07dlgfM2rUKE2ZMkVjxozRvn37tGzZMgUGBkqS+vXrp2XLliktLc06/qOPPlLZsmXVvHnzu/p5MbUYAAAAAAzWuXNnm/uLFy9WiRIltG/fPm3btk1nz57Vzp07VbRoUUlS5cqVrWMnTpyorl27asKECdZttWrVsjvDkCFD1KlTJ5ttw4YNs349aNAgbdiwQStXrlS9evX022+/adasWZozZ4569eolSapUqZIaN24sSerUqZMiIyP1xRdf6Mknn5QkLVmyRL17977rlaU5IgsAAAAABjt8+LC6deumihUrys/PT8HBwZKkpKQkxcfHq3bt2tYS+3fx8fFq0aLFXWcIDw+3uZ+RkaHXX39doaGhKlq0qAoXLqwNGzYoKSlJkrR//36lpaXd8rW9vLz09NNPa/HixZKkXbt26aefflLv3r3vOitHZAEAAADAYO3atVP58uW1YMEClSlTRpmZmapRo4bS09NVqFCh2z72n/ZbLBZlZWXZbLtx40a2cT4+Pjb3p02bplmzZmnmzJkKDQ2Vj4+PhgwZovT09Dt6XemP6cVhYWE6deqU3n//fTVv3lzly5f/x8f9E47IAgAAAICBzp8/r4MHD2r06NFq0aKFqlevrosXL1r316xZU/Hx8bpw4UKOj69Zs+ZtF08qUaKETp8+bb1/+PBhXbt27R9zxcTEqEOHDnrqqadUq1YtVaxYUYcOHbLur1KligoVKnTb1w4NDVV4eLgWLFigZcuWqW/fvv/4uneCIgsAAAAABipSpIiKFSum9957T0eOHNG3336rqKgo6/5u3bqpVKlS6tixo2JiYnT06FF99tlnio2NlSSNGzdOn3zyicaNG6f9+/dr7969+ve//219fPPmzTVnzhzt3r1bP/74owYMGHBHl9apUqWKvv76a23btk379+/Xc889p5SUFOt+Ly8vjRgxQi+//LI++OADJSYmavv27Vq0aJHN8/Tr109TpkxRVlaWzWrKd4MiCwAAAAAGcnNz0/LlyxUXF6caNWropZde0rRp06z7PTw8tHHjRpUsWVJt2rRRaGiopkyZInd3d0lSs2bN9Omnn+rLL79UWFiYmjdvbrOy8FtvvaWgoCA1adJE3bt317Bhw+Tt7f2PuUaPHq06deqoVatWatasmbVM/9WYMWM0dOhQjR07VtWrV1eXLl105swZmzHdunVTgQIF1K1bN3l5ed3FT+p/LFl/nyx9D7py5Yr8/f11+fJl+fn5GR0HAAAAgEFu1w2uX7+uY8eOqUKFCnlWmHD3jh8/rkqVKmnnzp2qU6fObcfe6XvIYk8AAAAAgDx348YNnT9/XqNHj9ZDDz30jyXWHrmaWjx37lwFBwfLy8tL9evXtzlsnZOZM2eqatWqKlSokIKCgvTSSy/p+vXruQoMAAAAALj3xcTEqHTp0tq5c6fmz5+fp89t9xHZFStWKCoqSvPnz1f9+vU1c+ZMtWrVSgcPHlTJkiWzjV+2bJlGjhypxYsXq2HDhjp06JD1ArjTp0/Pk2/CEOP97Rx/2TE5AAAA8lDwyLV2jT/u1d2u8aEVytk1XpJWTr5p1/jqB/bb/RoA8l6zZs2yXfYnr9h9RHb69Onq37+/+vTpo5CQEM2fP1/e3t7Wi9z+3bZt29SoUSN1795dwcHBatmypbp16/aPR3EBAAAAAMiJXUU2PT1dcXFxioiI+N8TuLkpIiLCuvTz3zVs2FBxcXHW4nr06FGtW7dObdq0ueXrpKWl6cqVKzY3AAAAAAAkO6cWnzt3ThkZGQoMDLTZHhgYqAMHDuT4mO7du+vcuXNq3LixsrKydPPmTQ0YMECvvPLKLV9n8uTJmjBhgj3RAAAAAAAuwuHXkd28ebMmTZqkd955R7t27dLq1au1du1avf7667d8zKhRo3T58mXr7eTJk46OCQAAAAAwCbuOyBYvXlzu7u5KSUmx2Z6SkqJSpUrl+JgxY8bo6aefVr9+/SRJoaGhunr1qp599lm9+uqrcnPL3qU9PT3l6elpT7S7Zv/CBvY9f+jSULvG7+21174XAAAAAAAXYdcRWQ8PD9WtW1fR0dHWbZmZmYqOjlaDBg1yfMy1a9eylVV3d3dJctgKVgAAAAAA52X31OKoqCgtWLBAS5cu1f79+zVw4EBdvXpVffr0kST17NlTo0aNso5v166d5s2bp+XLl+vYsWP6+uuvNWbMGLVr185aaAEAAAAA+Wfz5s2yWCy6dOlSno7NL3ZfR7ZLly46e/asxo4dq+TkZIWFhWn9+vXWBaCSkpJsjsCOHj1aFotFo0eP1i+//KISJUqoXbt2mjhxYt59F05of7Xqdo3nemkAAAAA7lTDhg11+vRp+fv75+nY/GJ3kZWkyMhIRUZG5rhv8+bNti9QoIDGjRuncePG5ealAAAAAODujM/nAjb+skOfPj09XR4eHnf1HB4eHrdc5+huxuYXh69aDAAAAAC4tWbNmlkPFvr7+6t48eIaM2aMdU2h4OBgvf766+rZs6f8/Pz07LPPSpK2bt2qJk2aqFChQgoKCtKLL76oq1evWp83LS1NI0aMUFBQkDw9PVW5cmUtWrRIUvbpwidOnFC7du1UpEgR+fj46IEHHtC6detyHCtJn332mR544AF5enoqODhYb731ls33FBwcrEmTJqlv377y9fVVuXLl9N577+XZz4wiCwAAAAAGW7p0qQoUKKAdO3Zo1qxZmj59uhYuXGjd/+abb6pWrVravXu3xowZo8TERD322GPq3Lmz9uzZoxUrVmjr1q02M2d79uypTz75RLNnz9b+/fv17rvvqnDhwjm+/gsvvKC0tDRt2bJFe/fu1b///e9bjo2Li9OTTz6prl27au/evRo/frzGjBmjJUuW2Ix76623FB4ert27d+v555/XwIEDdfDgwbv/YSmXU4sBAAAAAHknKChIM2bMkMViUdWqVbV3717NmDFD/fv3lyQ1b95cQ4cOtY7v16+fevTooSFDhkiSqlSpotmzZ6tp06aaN2+ekpKStHLlSn399deKiIiQJFWsWPGWr5+UlKTOnTsrNDT0H8dOnz5dLVq00JgxYyRJ999/v/bt26dp06apd+/e1nFt2rTR888/L0kaMWKEZsyYoU2bNqlq1ar2/4D+hiOyAAAAAGCwhx56SBaLxXq/QYMGOnz4sDIyMiRJ4eHhNuMTEhK0ZMkSFS5c2Hpr1aqVMjMzdezYMcXHx8vd3V1Nmza9o9d/8cUX9cYbb6hRo0YaN26c9uzZc8ux+/fvV6NGjWy2NWrUyCavJNWsWdP6tcViUalSpXTmzJk7yvNPOCKLe0bwyLV2jT/u1d2u8aEVytk1fm+vvXaNBwAAABzFx8fH5n5qaqqee+45vfjii9nGlitXTkeOHLHr+fv166dWrVpp7dq12rhxoyZPnqy33npLgwYNynXmggUL2ty3WCzKzMzM9fP9FUUWuAUugQQAAID88sMPP9jc3759u6pUqSJ3d/ccx9epU0f79u1T5cqVc9wfGhqqzMxMfffdd9apxf8kKChIAwYM0IABAzRq1CgtWLAgxyJbvXp1xcTE2GyLiYnR/ffff8u8eY0iCwAAANxD7rVZaisn37RrvMQH/LmRlJSkqKgoPffcc9q1a5fefvvtbCsB/9WIESP00EMPKTIyUv369ZOPj4/27dunr7/+WnPmzFFwcLB69eqlvn37avbs2apVq5ZOnDihM2fO6Mknn8z2fEOGDFHr1q11//336+LFi9q0aZOqV8/5wM7QoUP14IMP6vXXX1eXLl0UGxurOXPm6J133smzn8c/ocgCAAAAgMF69uyp33//XfXq1ZO7u7sGDx5svcxOTmrWrKnvvvtOr776qpo0aaKsrCxVqlRJXbp0sY6ZN2+eXnnlFT3//PM6f/68ypUrp1deeSXH58vIyNALL7ygU6dOyc/PT4899phmzJiR49g6depo5cqVGjt2rF5//XWVLl1ar732ms1CT45GkQUAAADg3MZfNjrBPypYsKBmzpypefPmZdt3/PjxHB/z4IMPauPGjbd8Ti8vL02fPl3Tp0/Ptq9Zs2bW69RK0ttvv33L5/n7WEnq3LmzOnfufMvH5JQ5Pj7+luPtxarFAAAAAABTocgCAAAAAEyFqcUAAAAAYKDNmzcbHcF0OCILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAADgYsaPH6+wsDDr/d69e6tjx46G5bEXRRYAAAAAYCoFjA4AAAAAAI4UujQ0X19vb6+9d/X49PR0eXh45FEa58QRWQAAAAAwULNmzRQZGakhQ4aoePHiatWqlX766Se1bt1ahQsXVmBgoJ5++mmdO3fO+pjMzExNnTpVlStXlqenp8qVK6eJEyda948YMUL333+/vL29VbFiRY0ZM0Y3btww4ttzCIosAAAAABhs6dKl8vDwUExMjKZMmaLmzZurdu3a+vHHH7V+/XqlpKToySeftI4fNWqUpkyZojFjxmjfvn1atmyZAgMDrft9fX21ZMkS7du3T7NmzdKCBQs0Y8YMI741h2BqMQAAAAAYrEqVKpo6daok6Y033lDt2rU1adIk6/7FixcrKChIhw4dUunSpTVr1izNmTNHvXr1kiRVqlRJjRs3to4fPXq09evg4GANGzZMy5cv18svv5xP35FjUWQBAAAAwGB169a1fp2QkKBNmzapcOHC2cYlJibq0qVLSktLU4sWLW75fCtWrNDs2bOVmJio1NRU3bx5U35+fg7JbgSKLAAAAAAYzMfHx/p1amqq2rVrp3//+9/ZxpUuXVpHjx697XPFxsaqR48emjBhglq1aiV/f38tX75cb731Vp7nNgpFFgAAAADuIXXq1NFnn32m4OBgFSiQvbJVqVJFhQoVUnR0tPr165dt/7Zt21S+fHm9+uqr1m0nTpxwaOb8xmJPAAAAAHAPeeGFF3ThwgV169ZNO3fuVGJiojZs2KA+ffooIyNDXl5eGjFihF5++WV98MEHSkxM1Pbt27Vo0SJJfxTdpKQkLV++XImJiZo9e7Y+//xzg7+rvEWRBQAAAIB7SJkyZRQTE6OMjAy1bNlSoaGhGjJkiAICAuTm9keFGzNmjIYOHaqxY8eqevXq6tKli86cOSNJat++vV566SVFRkYqLCxM27Zt05gxY4z8lvIcU4sBAAAAOLW9vfYaHeG2Nm/enG1blSpVtHr16ls+xs3NTa+++qrN9OG/mjp1qnUV5D8NGTLE+vX48eM1fvx46/0lS5bYE9lwHJEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmwmJPAKyCR661a/xxr+52jQ+tUM6u8Ssn37RrfPUD++0aDwAAAHPiiCwAAAAAwFQ4IgsAwN0Y72/n+MuOyQEAgAuhyALAPYTp3cay9+cvSce97BsfujTUrvGu9h4AAHAnmFoMAAAAADAViiwAAAAAGCgrK0vPPvusihYtKovFovj4eKMj3fMosgAAAABgoPXr12vJkiX6z3/+o9OnT+vKlStq166dypQpI4vFojVr1hgd8Z7DObIAAAAAnNr+atXz9fXsXa8gMTFRpUuXVsOGDSVJu3fvVq1atdS3b1916tTJERFNjyILAAAAAAbp3bu3li5dKkmyWCwqX768jh8/rtatWxuc7N5GkQUAAOZl7+WPJC6BBOCeMmvWLFWqVEnvvfeedu7cKXd3d6MjmQJFFgAA4DbsnZLIJZAA2MPf31++vr5yd3dXqVKljI5jGhRZAABwz7D/Wsr2v4bd1/K1/yUAAA7GqsUAAAAAAFOhyAIAAAAATIWpxQAAAABwD0lNTdWRI0es948dO6b4+HgVLVpU5cqVMzDZvYMiCwAAAAD3kB9//FGPPPKI9X5UVJQkqVevXlqyZIlBqe4tFFkAAAAATu1eX018yJAhGjJkiPV+s2bNlJWVZVwgE+AcWQAAAACAqVBkAQAAAACmkqsiO3fuXAUHB8vLy0v169fXjh07bjm2WbNmslgs2W5t27bNdWgAAAAAgOuyu8iuWLFCUVFRGjdunHbt2qVatWqpVatWOnPmTI7jV69erdOnT1tvP/30k9zd3fXEE0/cdXgAAAAAgOuxu8hOnz5d/fv3V58+fRQSEqL58+fL29tbixcvznF80aJFVapUKevt66+/lre3N0UWAAAAAJArdhXZ9PR0xcXFKSIi4n9P4OamiIgIxcbG3tFzLFq0SF27dpWPj88tx6SlpenKlSs2NwAAAAC4E6z4a153+t7ZVWTPnTunjIwMBQYG2mwPDAxUcnLyPz5+x44d+umnn9SvX7/bjps8ebL8/f2tt6CgIHtiAgAAAHBB7u7ukv44AAdzunbtmiSpYMGCtx2Xr9eRXbRokUJDQ1WvXr3bjhs1apT1or+SdOXKFcosAAAAgNsqUKCAvL29dfbsWRUsWFBublykxSyysrJ07do1nTlzRgEBAdYPJW7FriJbvHhxubu7KyUlxWZ7SkqKSpUqddvHXr16VcuXL9drr732j6/j6ekpT09Pe6IBAAAAcHEWi0WlS5fWsWPHdOLECaPjIBcCAgL+sVtKdhZZDw8P1a1bV9HR0erYsaMkKTMzU9HR0YqMjLztYz/99FOlpaXpqaeesuclAQAAAOCOeXh4qEqVKkwvNqGCBQv+45HYP9k9tTgqKkq9evVSeHi46tWrp5kzZ+rq1avq06ePJKlnz54qW7asJk+ebPO4RYsWqWPHjipWrJi9LwkAAAAAd8zNzU1eXl5Gx4AD2V1ku3TporNnz2rs2LFKTk5WWFiY1q9fb10AKikpKdtc9IMHD2rr1q3auHFj3qQGAAAAALisXC32FBkZecupxJs3b862rWrVqiyBDQAAAADIEyzjBQAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATCVXRXbu3LkKDg6Wl5eX6tevrx07dtx2/KVLl/TCCy+odOnS8vT01P33369169blKjAAAAAAwLUVsPcBK1asUFRUlObPn6/69etr5syZatWqlQ4ePKiSJUtmG5+enq5HH31UJUuW1KpVq1S2bFmdOHFCAQEBeZEfAAAAAOBi7C6y06dPV//+/dWnTx9J0vz587V27VotXrxYI0eOzDZ+8eLFunDhgrZt26aCBQtKkoKDg+8uNQAAAADAZdk1tTg9PV1xcXGKiIj43xO4uSkiIkKxsbE5PubLL79UgwYN9MILLygwMFA1atTQpEmTlJGRcXfJAQAAAAAuya4jsufOnVNGRoYCAwNttgcGBurAgQM5Pubo0aP69ttv1aNHD61bt05HjhzR888/rxs3bmjcuHE5PiYtLU1paWnW+1euXLEnJgAAAADAiTl81eLMzEyVLFlS7733nurWrasuXbro1Vdf1fz582/5mMmTJ8vf3996CwoKcnRMAAAAAIBJ2FVkixcvLnd3d6WkpNhsT0lJUalSpXJ8TOnSpXX//ffL3d3duq169epKTk5Wenp6jo8ZNWqULl++bL2dPHnSnpgAAAAAACdmV5H18PBQ3bp1FR0dbd2WmZmp6OhoNWjQIMfHNGrUSEeOHFFmZqZ126FDh1S6dGl5eHjk+BhPT0/5+fnZ3AAAAAAAkHIxtTgqKkoLFizQ0qVLtX//fg0cOFBXr161rmLcs2dPjRo1yjp+4MCBunDhggYPHqxDhw5p7dq1mjRpkl544YW8+y4AAAAAAC7D7svvdOnSRWfPntXYsWOVnJyssLAwrV+/3roAVFJSktzc/tePg4KCtGHDBr300kuqWbOmypYtq8GDB2vEiBF5910AAAAAAFyG3UVWkiIjIxUZGZnjvs2bN2fb1qBBA23fvj03LwUAAAAAgA2Hr1oMAAAAAEBeosgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTyVWRnTt3roKDg+Xl5aX69etrx44dtxy7ZMkSWSwWm5uXl1euAwMAAAAAXJvdRXbFihWKiorSuHHjtGvXLtWqVUutWrXSmTNnbvkYPz8/nT592no7ceLEXYUGAAAAALguu4vs9OnT1b9/f/Xp00chISGaP3++vL29tXjx4ls+xmKxqFSpUtZbYGDgXYUGAAAAALguu4psenq64uLiFBER8b8ncHNTRESEYmNjb/m41NRUlS9fXkFBQerQoYN+/vnn275OWlqarly5YnMDAAAAAECys8ieO3dOGRkZ2Y6oBgYGKjk5OcfHVK1aVYsXL9YXX3yhjz76SJmZmWrYsKFOnTp1y9eZPHmy/P39rbegoCB7YgIAAAAAnJjDVy1u0KCBevbsqbCwMDVt2lSrV69WiRIl9O67797yMaNGjdLly5ett5MnTzo6JgAAAADAJArYM7h48eJyd3dXSkqKzfaUlBSVKlXqjp6jYMGCql27to4cOXLLMZ6envL09LQnGgAAAADARdh1RNbDw0N169ZVdHS0dVtmZqaio6PVoEGDO3qOjIwM7d27V6VLl7YvKQAAAAAAsvOIrCRFRUWpV69eCg8PV7169TRz5kxdvXpVffr0kST17NlTZcuW1eTJkyVJr732mh566CFVrlxZly5d0rRp03TixAn169cvb78TAAAAAIBLsLvIdunSRWfPntXYsWOVnJyssLAwrV+/3roAVFJSktzc/neg9+LFi+rfv7+Sk5NVpEgR1a1bV9u2bVNISEjefRcAAAAAAJdhd5GVpMjISEVGRua4b/PmzTb3Z8yYoRkzZuTmZQAAAAAAyMbhqxYDAAAAAJCXKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUclVk586dq+DgYHl5eal+/frasWPHHT1u+fLlslgs6tixY25eFgAAAAAA+4vsihUrFBUVpXHjxmnXrl2qVauWWrVqpTNnztz2ccePH9ewYcPUpEmTXIcFAAAAAMDuIjt9+nT1799fffr0UUhIiObPny9vb28tXrz4lo/JyMhQjx49NGHCBFWsWPGuAgMAAAAAXJtdRTY9PV1xcXGKiIj43xO4uSkiIkKxsbG3fNxrr72mkiVL6plnnrmj10lLS9OVK1dsbgAAAAAASHYW2XPnzikjI0OBgYE22wMDA5WcnJzjY7Zu3apFixZpwYIFd/w6kydPlr+/v/UWFBRkT0wAAAAAgBNz6KrFv/32m55++mktWLBAxYsXv+PHjRo1SpcvX7beTp486cCUAAAAAAAzKWDP4OLFi8vd3V0pKSk221NSUlSqVKls4xMTE3X8+HG1a9fOui0zM/OPFy5QQAcPHlSlSpWyPc7T01Oenp72RAMAAAAAuAi7jsh6eHiobt26io6Otm7LzMxUdHS0GjRokG18tWrVtHfvXsXHx1tv7du31yOPPKL4+HimDAMAAAAA7GbXEVlJioqKUq9evRQeHq569epp5syZunr1qvr06SNJ6tmzp8qWLavJkyfLy8tLNWrUsHl8QECAJGXbDgAAAADAnbC7yHbp0kVnz57V2LFjlZycrLCwMK1fv966AFRSUpLc3Bx66i0AAAAAwIXZXWQlKTIyUpGRkTnu27x5820fu2TJkty8JAAAAAAAkhy8ajEAAAAAAHmNIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTyVWRnTt3roKDg+Xl5aX69etrx44dtxy7evVqhYeHKyAgQD4+PgoLC9OHH36Y68AAAAAAANdmd5FdsWKFoqKiNG7cOO3atUu1atVSq1atdObMmRzHFy1aVK+++qpiY2O1Z88e9enTR3369NGGDRvuOjwAAAAAwPXYXWSnT5+u/v37q0+fPgoJCdH8+fPl7e2txYsX5zi+WbNmevzxx1W9enVVqlRJgwcPVs2aNbV169a7Dg8AAAAAcD12Fdn09HTFxcUpIiLif0/g5qaIiAjFxsb+4+OzsrIUHR2tgwcP6uGHH7Y/LQAAAADA5RWwZ/C5c+eUkZGhwMBAm+2BgYE6cODALR93+fJllS1bVmlpaXJ3d9c777yjRx999Jbj09LSlJaWZr1/5coVe2ICAAAAAJyYXUU2t3x9fRUfH6/U1FRFR0crKipKFStWVLNmzXIcP3nyZE2YMCE/ogEAAAAATMauIlu8eHG5u7srJSXFZntKSopKlSp1y8e5ubmpcuXKkqSwsDDt379fkydPvmWRHTVqlKKioqz3r1y5oqCgIHuiAgAAAACclF3nyHp4eKhu3bqKjo62bsvMzFR0dLQaNGhwx8+TmZlpM3X47zw9PeXn52dzAwAAAABAysXU4qioKPXq1Uvh4eGqV6+eZs6cqatXr6pPnz6SpJ49e6ps2bKaPHmypD+mCYeHh6tSpUpKS0vTunXr9OGHH2revHl5+50AAAAAAFyC3UW2S5cuOnv2rMaOHavk5GSFhYVp/fr11gWgkpKS5Ob2vwO9V69e1fPPP69Tp06pUKFCqlatmj766CN16dIl774LAAAAAIDLyNViT5GRkYqMjMxx3+bNm23uv/HGG3rjjTdy8zIAAAAAAGRj1zmyAAAAAAAYjSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMJVdFdu7cuQoODpaXl5fq16+vHTt23HLsggUL1KRJExUpUkRFihRRRETEbccDAAAAAHA7dhfZFStWKCoqSuPGjdOuXbtUq1YttWrVSmfOnMlx/ObNm9WtWzdt2rRJsbGxCgoKUsuWLfXLL7/cdXgAAAAAgOuxu8hOnz5d/fv3V58+fRQSEqL58+fL29tbixcvznH8xx9/rOeff15hYWGqVq2aFi5cqMzMTEVHR991eAAAAACA67GryKanpysuLk4RERH/ewI3N0VERCg2NvaOnuPatWu6ceOGihYtessxaWlpunLlis0NAAAAAADJziJ77tw5ZWRkKDAw0GZ7YGCgkpOT7+g5RowYoTJlytiU4b+bPHmy/P39rbegoCB7YgIAAAAAnFi+rlo8ZcoULV++XJ9//rm8vLxuOW7UqFG6fPmy9Xby5Ml8TAkAAAAAuJcVsGdw8eLF5e7urpSUFJvtKSkpKlWq1G0f++abb2rKlCn65ptvVLNmzduO9fT0lKenpz3RAAAAAAAuwq4jsh4eHqpbt67NQk1/LtzUoEGDWz5u6tSpev3117V+/XqFh4fnPi0AAAAAwOXZdURWkqKiotSrVy+Fh4erXr16mjlzpq5evao+ffpIknr27KmyZctq8uTJkqR///vfGjt2rJYtW6bg4GDrubSFCxdW4cKF8/BbAQAAAAC4AruLbJcuXXT27FmNHTtWycnJCgsL0/r1660LQCUlJcnN7X8HeufNm6f09HT93//9n83zjBs3TuPHj7+79AAAAAAAl2N3kZWkyMhIRUZG5rhv8+bNNvePHz+em5cAAAAAACBH+bpqMQAAAAAAd4siCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATCVXRXbu3LkKDg6Wl5eX6tevrx07dtxy7M8//6zOnTsrODhYFotFM2fOzG1WAAAAAADsL7IrVqxQVFSUxo0bp127dqlWrVpq1aqVzpw5k+P4a9euqWLFipoyZYpKlSp114EBAAAAAK7N7iI7ffp09e/fX3369FFISIjmz58vb29vLV68OMfxDz74oKZNm6auXbvK09PzrgMDAAAAAFybXUU2PT1dcXFxioiI+N8TuLkpIiJCsbGxeRYqLS1NV65csbkBAAAAACDZWWTPnTunjIwMBQYG2mwPDAxUcnJynoWaPHmy/P39rbegoKA8e24AAAAAgLndk6sWjxo1SpcvX7beTp48aXQkAAAAAMA9ooA9g4sXLy53d3elpKTYbE9JScnThZw8PT05nxYAAAAAkCO7jsh6eHiobt26io6Otm7LzMxUdHS0GjRokOfhAAAAAAD4O7uOyEpSVFSUevXqpfDwcNWrV08zZ87U1atX1adPH0lSz549VbZsWU2ePFnSHwtE7du3z/r1L7/8ovj4eBUuXFiVK1fOw28FAAAAAOAK7C6yXbp00dmzZzV27FglJycrLCxM69evty4AlZSUJDe3/x3o/fXXX1W7dm3r/TfffFNvvvmmmjZtqs2bN9/9dwAAAAAAcCl2F1lJioyMVGRkZI77/l5Og4ODlZWVlZuXAQAAAAAgm3ty1WIAAAAAAG6FIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTyVWRnTt3roKDg+Xl5aX69etrx44dtx3/6aefqlq1avLy8lJoaKjWrVuXq7AAAAAAANhdZFesWKGoqCiNGzdOu3btUq1atdSqVSudOXMmx/Hbtm1Tt27d9Mwzz2j37t3q2LGjOnbsqJ9++umuwwMAAAAAXI/dRXb69Onq37+/+vTpo5CQEM2fP1/e3t5avHhxjuNnzZqlxx57TMOHD1f16tX1+uuvq06dOpozZ85dhwcAAAAAuB67imx6erri4uIUERHxvydwc1NERIRiY2NzfExsbKzNeElq1arVLccDAAAAAHA7BewZfO7cOWVkZCgwMNBme2BgoA4cOJDjY5KTk3Mcn5ycfMvXSUtLU1pamvX+5cuXJUlXrlyxJ65dMtOu2TX+iiXLrvEZv2fYNT41w77xjvzZ5BfeA+PxHhiP98BY9v78Jd6DvObovwMS78E/udf+HZJ4D/7JvfbvkOTY9+DP587Ksv/vP5yHXUU2v0yePFkTJkzItj0oKMiANDnzt/sR++0aXc/ep/e3P5HZ8R4Yj/fAeLwHxuM9MFbuvlveg7zk6L8DEu/BP7nn/h2S8uU9+O233+TvYu81/seuIlu8eHG5u7srJSXFZntKSopKlSqV42NKlSpl13hJGjVqlKKioqz3MzMzdeHCBRUrVkwWi8WeyPeEK1euKCgoSCdPnpSfn5/RcVwS74HxeA+Mx3tgPN4D4/EeGIufv/Gc4T3IysrSb7/9pjJlyhgdBQayq8h6eHiobt26io6OVseOHSX9UTKjo6MVGRmZ42MaNGig6OhoDRkyxLrt66+/VoMGDW75Op6envL09LTZFhAQYE/Ue5Kfn59p/8FwFrwHxuM9MB7vgfF4D4zHe2Asfv7GM/t7wJFY2D21OCoqSr169VJ4eLjq1aunmTNn6urVq+rTp48kqWfPnipbtqwmT54sSRo8eLCaNm2qt956S23bttXy5cv1448/6r333svb7wQAAAAA4BLsLrJdunTR2bNnNXbsWCUnJyssLEzr16+3LuiUlJQkN7f/LYbcsGFDLVu2TKNHj9Yrr7yiKlWqaM2aNapRo0befRcAAAAAAJeRq8WeIiMjbzmVePPmzdm2PfHEE3riiSdy81JOwdPTU+PGjcs2XRr5h/fAeLwHxuM9MB7vgfF4D4zFz994vAdwFpYs1q0GAAAAAJiI2z8PAQAAAADg3kGRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAOAQN27cUN++fXXs2DGjowBwMqxa7GDp6ek6duyYKlWqpAIFcnW1I8DUdu7cqU2bNunMmTPKzMy02Td9+nSDUgEA8ou/v7/i4+NVoUIFo6MAcCI0Kwe5du2aBg0apKVLl0qSDh06pIoVK2rQoEEqW7asRo4caXBC15GYmKj3339fiYmJmjVrlkqWLKn//ve/KleunB544AGj4zm1SZMmafTo0apataoCAwNlsVis+/76NeBMOnXqdMdjV69e7cAkru3LL7+847Ht27d3YBJ07NhRa9as0UsvvWR0FJfl7u6u06dPq2TJkjbbz58/r5IlSyojI8OgZEDuUWQdZNSoUUpISNDmzZv12GOPWbdHRERo/PjxFNl88t1336l169Zq1KiRtmzZookTJ6pkyZJKSEjQokWLtGrVKqMjOrVZs2Zp8eLF6t27t9FRXFaFChVu+6HB0aNH8zGNa/D39zc6AvRHefori8Wiv05C++vfC36Jd6wqVarotddeU0xMjOrWrSsfHx+b/S+++KJByVzHrSZgpqWlycPDI5/TAHmDIusga9as0YoVK/TQQw/Z/Gf5wAMPKDEx0cBkrmXkyJF64403FBUVJV9fX+v25s2ba86cOQYmcw1ubm5q1KiR0TFc2pAhQ2zu37hxQ7t379b69es1fPhwY0I5uffff9/oCJBsTmX45ptvNGLECE2aNEkNGjSQJMXGxmr06NGaNGmSURFdxqJFixQQEKC4uDjFxcXZ7LNYLBRZB5o9e7akP37OCxcuVOHCha37MjIytGXLFlWrVs2oeMBdocg6yNmzZ7NN35Ckq1evMqUyH+3du1fLli3Ltr1kyZI6d+6cAYlcy0svvaS5c+dq5syZRkdxWYMHD85x+9y5c/Xjjz/mcxrAGEOGDNH8+fPVuHFj67ZWrVrJ29tbzz77rPbv329gOufHQk/GmTFjhqQ/jsjOnz9f7u7u1n0eHh4KDg7W/PnzjYoH3BWKrIOEh4dr7dq1GjRokKT/TWFauHCh9dNgOF5AQIBOnz6dbYGJ3bt3q2zZsgalch3Dhg1T27ZtValSJYWEhKhgwYI2+zk/0DitW7fWqFGjOHqYD1atWqWVK1cqKSlJ6enpNvt27dplUCrXkpiYqICAgGzb/f39dfz48XzP46pYADP//fkhwiOPPKLVq1erSJEiBicC8g6X33GQSZMm6ZVXXtHAgQN18+ZNzZo1Sy1bttT777+viRMnGh3PZXTt2lUjRoxQcnKyLBaLMjMzFRMTo2HDhqlnz55Gx3N6L774ojZt2qT7779fxYoVk7+/v80Nxlm1apWKFi1qdAynN3v2bPXp00eBgYHavXu36tWrp2LFiuno0aNq3bq10fFcxoMPPqioqCilpKRYt6WkpGj48OGqV6+egclcw7Vr1/TMM8/I29tbDzzwgJKSkiRJgwYN0pQpUwxO5xo2bdpEiYXT4fI7DpSYmKgpU6YoISFBqampqlOnjkaMGKHQ0FCjo7mM9PR0vfDCC1qyZIkyMjJUoEABZWRkqHv37lqyZInNFBvkPV9fXy1fvlxt27Y1OorLql27ts3pDFlZWUpOTtbZs2f1zjvv6NlnnzUwnfOrVq2axo0bp27dusnX11cJCQmqWLGixo4dqwsXLnCufj45cuSIHn/8cR06dEhBQUGSpJMnT6pKlSpas2aNKleubHBC5zZ48GDFxMRo5syZeuyxx7Rnzx5VrFhRX3zxhcaPH6/du3cbHdHpZWRkaMmSJYqOjs7xcnjffvutQcmA3KPIwiWcPHlSe/fuVWpqqmrXrq0qVaoYHckllC9fXhs2bGAhCQNNmDDB5r6bm5tKlCihZs2a8b7kA29vb+3fv1/ly5dXyZIl9fXXX6tWrVo6fPiwHnroIZ0/f97oiC4jKytLX3/9tQ4cOCBJql69uiIiIli3Ih+UL1/eugDmXz/QOXLkiOrUqaMrV64YHdHpRUZGasmSJWrbtq1Kly6d7c/9n+fSAmbCCQoOlJmZqSNHjuT4ydfDDz9sUCrXFBQUpKCgIGVkZGjv3r26ePEiU2zywfjx4zVu3Di9//778vb2NjqOy7l586YqVKigVq1aKTAw0Og4LqlUqVK6cOGCypcvr3Llymn79u2qVauWjh07dsvLYcAxLBaLWrZsqZYtWxodxeWwAKbxli9frpUrV6pNmzZGRwHyDEXWQbZv367u3bvrxIkT2X5ZsVgsXLMunwwZMkShoaF65plnlJGRoaZNm2rbtm3y9vbWf/7zHzVr1szoiE5t9uzZSkxMVGBgoIKDg7Mt9sRCN45VoEABDRgwgBVZDdS8eXN9+eWXql27tvr06aOXXnpJq1at0o8//qhOnToZHc/lpaSk6N1339XYsWONjuLUWADTeB4eHkyhh9OhyDrIgAEDrP9w5zSFA/lj1apVeuqppyRJX331lY4ePaoDBw7oww8/1KuvvqqYmBiDEzq3Dh068GffYPXq1dPu3btVvnx5o6O4pPfee886I+eFF15QsWLFtG3bNrVv317PPfecwemQnJysCRMmUGQdbNKkSWrdurX27dtnXQBz37592rZtm7777juj47mEoUOHatasWZozZw7/L8NpcI6sg/j4+CghIYFPvwzm5eWlI0eO6L777tOzzz4rb29vzZw5U8eOHVOtWrU4LwdOb+XKlRo1apReeukl1a1bVz4+Pjb7a9asaVAywPH27Nlz2/0HDhxQt27dmCWVD1gA01iPP/64Nm3apKJFi+qBBx7gcnhwChyRdZD69evryJEjFFmDBQYGat++fSpdurTWr1+vefPmSfrjUgCsWOx4FStW1M6dO1WsWDGb7ZcuXVKdOnV09OhRg5K5jq5du0r641JIf7JYLMrKyuI0h3xy6dIl7dixI8f1ErgMmGOFhYVZ/7z/3V//HsDxKlWqpAULFhgdw2UFBATo8ccfNzoGkKcosg4yaNAgDR06VMnJyQoNDc32yRdHQfJHnz599OSTT1qnd0dEREiSfvjhB1ZszQfHjx/PsSilpaXp1KlTBiRyPceOHTM6gkv76quv1KNHD6WmpsrPz8+mNFksFoqsgxUtWlRTp05VixYtctz/888/q127dvmcyjXYM+PJz8/PgUkgSe+//77REYA8R5F1kM6dO0uS+vbta93GUZD8N378eNWoUUMnT57UE088IU9PT0mSu7u7Ro4caXA65/Xll19av96wYYP8/f2t9zMyMhQdHa0KFSoYEc3lcG6ssYYOHaq+fftq0qRJrNxtgLp16+rXX3+95d+DS5cusXq0gwQEBNzx0W5+J8ofN2/e1ObNm5WYmKju3bvL19dXv/76q/z8/FS4cGGj4wF24xxZBzlx4sRt9/PLJZyZm5ubJOU4pa9gwYIKDg7WW2+9pX/9619GxHMpf/1Q4a8sFou8vLxUuXJlPlRwIB8fH+3du1cVK1Y0OopL+vzzz3X16lXron9/d/HiRX355Zfq1atXPidzfn9dxOn48eMaOXKkevfubV2lODY2VkuXLtXkyZP5+eeDEydO6LHHHlNSUpLS0tJ06NAhVaxYUYMHD1ZaWprmz59vdETAbhRZOL3o6GhFR0fneH7a4sWLDUrlGipUqKCdO3eqePHiRkdxWW5ubjl+oPDXGSKNGzfWmjVruLayA3Tq1Eldu3bVk08+aXQUwDAtWrRQv3791K1bN5vty5Yt03vvvafNmzcbE8yFdOzYUb6+vlq0aJGKFSumhIQEVaxYUZs3b1b//v11+PBhoyMCdmNqcR768ssv1bp1axUsWPCWR0H+1L59+3xK5domTJig1157TeHh4VwGyQA5nZ956dIlBQQE5H8YF/X111/r1Vdf1cSJE1WvXj1J0o4dOzRmzBiNHj1a/v7+eu655zRs2DAtWrTI4LTOp23btho+fLj27duX43oJ/F+QP7Zu3arGjRsbHcNlxcbG5njELzw8XP369TMgkev5/vvvtW3bNnl4eNhsDw4O1i+//GJQKuDucEQ2D7m5uSk5OVklS5a0Tq3MCefI5p/SpUtr6tSpevrpp42O4pL+/e9/Kzg4WF26dJEkPfHEE/rss89UunRprVu3TrVq1TI4ofOrUaOG3nvvPTVs2NBme0xMjJ599ln9/PPP+uabb9S3b18lJSUZlNJ58X/BvcHDw0Nly5ZVt27d9NRTTykkJMToSC6latWq6tChg6ZOnWqz/eWXX9YXX3yhgwcPGpTMdRQpUkQxMTEKCQmRr6+v9Yjs1q1b1blzZ6WkpBgdEbDbrf+Hhd0yMzNVsmRJ69e3uvGLS/5JT0/P9gs88s/8+fMVFBQk6Y8jg998843Wr1+v1q1ba/jw4Qancw2JiYk5rgjq5+dnvfxRlSpVdO7cufyO5hL4v+De8Ouvv2ro0KH67rvvVKNGDYWFhWnatGmsnp5PZsyYobfffluhoaHq16+f+vXrp5o1a+rtt9/WjBkzjI7nElq2bKmZM2da71ssFqWmpmrcuHFq06aNccGAu8ARWTi1ESNGqHDhwhozZozRUVxSoUKFdOjQIQUFBWnw4MG6fv263n33XR06dEj169fXxYsXjY7o9Bo3bixfX1998MEHKlGihCTp7Nmz6tmzp65evaotW7bom2++0QsvvMBREbiEY8eOadmyZfrkk0904MABPfzww/r222+NjuX0Tp06pXfeeUcHDhyQJFWvXl0DBgywftgJxzp16pRatWqlrKwsHT58WOHh4Tp8+LCKFy+uLVu2WA/EAGZCkc1Ds2fPvuOxL774ogOT4E+DBw/WBx98oJo1a6pmzZrZzk+bPn26QclcQ5kyZbRq1So1bNhQVatW1RtvvKEnnnhCBw8e1IMPPmjXdQaROwcPHlSHDh107Ngx6y+MJ0+eVMWKFfXFF1/o/vvv15o1a/Tbb78xBd9BvvvuO7355pvav3+/JCkkJETDhw9XkyZNDE7mujIyMvTf//5XY8aM0Z49ezg6Dpdw8+ZNrVixQgkJCUpNTVWdOnXUo0cPFSpUyOhoQK5QZPPQnV7CwmKxWKf0wbEeeeSRW+6zWCx8Cu9gkZGR+s9//qMqVapo9+7dOn78uAoXLqzly5dr6tSp2rVrl9ERXUJmZqY2btyoQ4cOSfrjfLVHH330tudvIm989NFH6tOnjzp16qRGjRpJ+uP85M8//1xLlixR9+7dDU7oWmJiYvTxxx9r1apVun79ujp06KAePXroscceMzqa07t06ZIWLVpk/UDngQceUN++fW2uMw4A9qDIAnCYGzduaNasWTp58qR69+6t2rVrS/rjfClfX19Wq7yHhIaGat26dUzzy2PVq1fXs88+q5deeslm+/Tp07VgwQLrL/VwrFGjRmn58uX69ddf9eijj6pHjx7q0KGDvL29jY7mEn788Ue1atVKhQoVsq6evnPnTv3+++/auHGj6tSpY3BC5zd58mQFBgaqb9++NtsXL16ss2fPasSIEQYlA3KPIutg6enpOnbsmCpVqqQCBbjakVGOHDmixMREPfzwwypUqJD1+pkA/vDXVSyRdzw9PfXzzz+rcuXKNtuPHDmiGjVq6Pr16wYlcy2NGjVSjx499OSTT3JdawM0adJElStX1oIFC6y/C928eVP9+vXT0aNHtWXLFoMTOr/g4GAtW7Ys2wKYP/zwg7p27Zrj5fKAex3NykGuXbumQYMGaenSpZKkQ4cOqWLFiho0aJDKli2rkSNHGpzQNZw/f15PPvmkNm3aJIvFosOHD6tixYp65plnVKRIEb311ltGR3Q6/3QN5b/iGppwdkFBQYqOjs5WZL/55huOfuejmJgYoyO4tB9//NGmxEpSgQIF9PLLLys8PNzAZK4jOTlZpUuXzra9RIkSOn36tAGJgLtHkXWQUaNGKSEhQZs3b7Y59yYiIkLjx4+nyOaTl156SQULFlRSUpKqV69u3d6lSxdFRUVRZB2gY8eOdzSOa2jCFQwdOlQvvvii4uPjrUdCYmJitGTJEs2aNcvgdK7l8OHD2rRpk86cOaPMzEybfWPHjjUolWvw8/NTUlKSqlWrZrP95MmT8vX1NSiVawkKClJMTEy29VxiYmJUpkwZg1IBd4ci6yBr1qzRihUr9NBDD9lMYX3ggQeUmJhoYDLXsnHjRm3YsEH33XefzfYqVaroxIkTBqVybn//BRFwZQMHDlSpUqX01ltvaeXKlZL+OG92xYoV6tChg8HpXMeCBQs0cOBAFS9eXKVKlbL5f9lisVBkHaxLly565pln9Oabb9p8oDN8+HB169bN4HSuoX///hoyZIhu3Lih5s2bS5Kio6P18ssva+jQoQanA3KHIusgZ8+ezfGaXFevXuXczHx09erVHBfzuHDhgjw9PQ1IhJyw0BCc2eOPP67HH3/c6Bgu7Y033tDEiRNZ0MYgb775piwWi3r27KmbN29KkgoWLKiBAwdqypQpBqdzDcOHD9f58+f1/PPPKz09XZLk5eWlESNGaNSoUQanA3KHay84SHh4uNauXWu9/2d5XbhwoRo0aGBULJfTpEkTffDBB9b7FotFmZmZmjp16m0vzYP8dfz4cd24ccPoGACc1MWLF/XEE08YHcNleXh4aNasWbp48aLi4+MVHx+vCxcuaMaMGXyonA8yMjL0/fffa+TIkTp79qy2b9+uhIQEXbhwgdkIMDWOyDrIpEmT1Lp1a+3bt083b97UrFmztG/fPm3btk3fffed0fFcxtSpU9WiRQv9+OOPSk9P18svv6yff/5ZFy5cYPEPOL39+/dr+/btatCggapVq6YDBw5o1qxZSktL01NPPWWdXiZJ7777rgIDAw1M6zyKFi2qQ4cOqXjx4ipSpMhtZ+FcuHAhH5O5rieeeEIbN27UgAEDjI7iki5fvqyMjAwVLVpUoaGh1u0XLlxQgQIF5OfnZ2A65+fu7q6WLVtq//79qlChgh588EGjIwF5giLrII0bN1Z8fLymTJmi0NBQ63XSYmNjbf4Rh2PVqFFDhw4d0pw5c+Tr66vU1FR16tRJL7zwQo6r9wHOYv369erQoYMKFy6sa9eu6fPPP1fPnj1Vq1YtZWZmqmXLltq4caO1zHbv3t3gxM7jz+sk//k1p5MYr3LlyhozZoy2b9+u0NBQFSxY0Gb/iy++aFAy19C1a1e1a9dOzz//vM32lStX6ssvv9S6desMSuY6atSooaNHj2Zb7AkwM64jC8BwXMM07zVs2FDNmzfXG2+8oeXLl+v555/XwIEDNXHiREl/rKweFxenjRs3GpwUcLzb/fJusVh09OjRfEzjeooWLaqYmBibqwdI0oEDB9SoUSOdP3/eoGSuY/369Ro1apRef/111a1bVz4+Pjb7OSoOM6LIOsiuXbtUsGBB69HXL774Qu+//75CQkI0fvx4eXh4GJzQee3Zs+eOx9asWdOBSXCnKLJ5z9/fX3FxcapcubIyMzPl6empHTt2qHbt2pKkn376SREREUpOTjY4qXNzd3fX6dOnsy3+d/78eZUsWZJLUMEl+Pj4WI+G/9XevXtVv359Xbt2zaBkrsPN7X/L4vx1lkhWVhaXw4NpMbXYQZ577jmNHDlSoaGhOnr0qLp06aJOnTrp008/1bVr1zRz5kyjIzqtsLAwWSwW6z/Of/rzM5u/buMfbjizP/+su7m5ycvLS/7+/tZ9vr6+unz5slHRXMatPitOS0vjA02D5PR/ARyrXr16eu+99/T222/bbJ8/f77q1q1rUCrXsmnTJqMjAHmOIusghw4dUlhYmCTp008/VdOmTbVs2TLFxMSoa9euFFkHOnbsmPXr3bt3a9iwYRo+fLh1tejY2Fi99dZbmjp1qlER8TcsNJT3goODdfjwYVWqVEnSH3/uy5UrZ92flJTEeeIONHv2bEl/lKWFCxeqcOHC1n0ZGRnasmWLqlWrZlQ8l/TBBx9o2rRpOnz4sCTp/vvv1/Dhw/X0008bnMz5vfHGG4qIiFBCQoJatGgh6Y9rmO7cuZPTG/JJ06ZNjY4A5DmKrINkZWUpMzNTkvTNN9/oX//6lyQpKChI586dMzKa0ytfvrz16yeeeEKzZ89WmzZtrNtq1qypoKAgjRkzRh07djQgoWv4/fffFRcXp6JFiyokJMRm3/Xr17Vy5Ur17NlTEgsNOcLAgQNtZhzUqFHDZv9///tfm1WLkbdmzJgh6Y//C+bPny93d3frPg8PDwUHB2v+/PlGxXM506dP15gxYxQZGalGjRpJkrZu3aoBAwbo3LlzeumllwxO6NwaNWqk2NhYTZs2TStXrlShQoVUs2ZNLVq0SFWqVDE6nsv4/vvv9e677+ro0aP69NNPVbZsWX344YeqUKGCGjdubHQ8wG6cI+sgzZs3V1BQkCIiIvTMM89o3759qly5sr777jv16tVLx48fNzqiSyhUqJB27dqVbYGJ/fv3q06dOvr9998NSubcDh06pJYtWyopKUkWi0WNGzfW8uXLrUcAU1JSVKZMGaZ2w+k98sgjWr16tYoUKWJ0FJdWoUIFTZgwwfrh2Z+WLl2q8ePH28zkAZzRZ599pqefflo9evTQhx9+qH379qlixYqaM2eO1q1bx8rRMCW3fx6C3Jg5c6Z27dqlyMhIvfrqq6pcubIkadWqVWrYsKHB6VxH9erVNXnyZKWnp1u3paena/LkydnKLfLOiBEjVKNGDZ05c0YHDx6Ur6+vGjVqpKSkJKOjAflq06ZNlNh7wOnTp3P8v7dhw4Y6ffq0AYlcT2Zmpg4dOqStW7dqy5YtNjc43htvvKH58+drwYIFNpefatSokXbt2mVgMiD3OCKbz65fvy53d/ds17CDY+zYsUPt2rVTVlaWdYXiPXv2yGKx6KuvvlK9evUMTuicAgMD9c0331hXqMzKytLzzz+vdevWadOmTfLx8eGILFxC586dVa9ePY0YMcJm+9SpU7Vz5059+umnBiVzLTVq1FD37t31yiuv2Gx/4403tGLFCu3du9egZK5h+/bt6t69u06cOJFtATRWzM0f3t7e2rdvn4KDg22uFHD06FGFhITo+vXrRkcE7MY5svnMy8vL6AgupV69ejp69Kg+/vhjHThwQJLUpUsXde/ePds11JB3fv/9dxUo8L9/XiwWi+bNm6fIyEjrwmeAK9iyZYvGjx+fbXvr1q311ltv5X8gFzVhwgR16dJFW7ZssZ4jGxMTo+joaK1cudLgdM5vwIABCg8P19q1a1W6dGlWjDZAqVKldOTIEQUHB9ts37p1K5e+g2lRZB0kIyNDM2bM0MqVK5WUlGQztVWSLly4YFAy1+Pj46Nnn332tmPatm2rhQsXsoprHqlWrZp+/PHHbNO358yZI0lq3769EbGAfJeamprjZXYKFiyoK1euGJDINXXu3Fk//PCDZsyYoTVr1kj649STv15bGY5z+PBhrVq1ynqaFfJf//79NXjwYC1evFgWi0W//vqrYmNjNWzYMI0ZM8boeECucI6sg0yYMEHTp09Xly5ddPnyZUVFRalTp05yc3PL8dN5GGvLli0s/JSHHn/8cX3yySc57pszZ466det2y+trAs4kNDRUK1asyLZ9+fLl2VbzhmPVrVtXH330keLi4hQXF6ePPvqIEptP6tevryNHjhgdw6WNHDlS3bt3V4sWLZSamqqHH35Y/fr103PPPadBgwYZHQ/IFc6RdZBKlSpp9uzZatu2rXx9fRUfH2/dtn37dqZW3mP+er4IAOSVr776Sp06dVL37t2tlzuKjo7WJ598ok8//ZRLgOWTdevWyd3dXa1atbLZvmHDBmVmZqp169YGJXMNn3/+uUaPHq3hw4crNDQ02zohf65hAcdLT0/XkSNHlJqaqpCQEJtrXANmQ5F1EB8fH+3fv1/lypVT6dKltXbtWtWpU0dHjx5V7dq1dfnyZaMj4i8osgAcZe3atZo0aZLi4+Ot188cN26cmjZtanQ0l1GzZk1NmTLF5prikrR+/XqNGDFCCQkJBiVzDW5u2ScAWiwWZWVlsdiTAU6ePClJCgoKMjgJcHc4R9ZB7rvvPp0+fVrlypVTpUqVtHHjRtWpU0c7d+6Up6en0fEAAPmkbdu2atu2rdExXNrhw4dznMpdrVo1przmA67Ta7ybN29qwoQJmj17tlJTUyVJhQsX1qBBgzRu3DiupgFTosg6yOOPP67o6GjVr19fgwYN0lNPPaVFixYpKSlJL730ktHxAAD55NKlS1q1apWOHj2qYcOGqWjRotq1a5cCAwNVtmxZo+O5BH9/fx09ejTbiq1HjhxhBft8UL58eaMjuLxBgwZp9erVmjp1qho0aCBJio2N1fjx43X+/HnNmzfP4ISA/ZhanE9iY2MVGxurKlWqqF27dkbHwd8wtRiAI+zZs0cRERHy9/fX8ePHdfDgQVWsWFGjR49WUlKSPvjgA6MjuoTnnntOsbGx+vzzz1WpUiVJf5TYzp0768EHH9TChQsNTuj8PvzwQ82fP1/Hjh1TbGysypcvr5kzZ6pChQrq0KGD0fGcnr+/v5YvX57tfPB169apW7dunPIGU2LV4nzSoEEDRUVFUWLvUa+88oqKFi1qdAwATiYqKkq9e/fW4cOHba4j3qZNG23ZssXAZK5l6tSp8vHxUbVq1VShQgVVqFBB1atXV7FixfTmm28aHc/pzZs3T1FRUWrTpo0uXbpkPSc2ICBAM2fONDaci/D09Mw2I0GSKlSokOMlwgAz4IisAx08eFBvv/229u/fL+mPa9YNGjRIVatWNTiZa+FTYABG8ff3165du1SpUiWbmR8nTpxQ1apVdf36daMjuoysrCx9/fXXSkhIsC669fDDDxsdyyWEhIRo0qRJ6tixo83fg59++knNmjXTuXPnjI7o9F577TUdOHBA77//vnWtlrS0ND3zzDOqUqWKxo0bZ3BCwH6cI+sgn332mbp27arw8HDruQjbt29XjRo1tHz5cnXu3NnghK5h3rx5Gjt2rIYMGaKJEydm+xSYIgvAkTw9PXXlypVs2w8dOqQSJUoYkMh1WSwWtWzZUi1btrzlmNDQUK1bt47VXPPYsWPHcrxmr6enp65evWpAIteze/duRUdH67777lOtWrUkSQkJCUpPT1eLFi3UqVMn69jVq1cbFROwC0XWQV5++WWNGjVKr732ms32cePG6eWXX6bI5pO3335bCxYsUMeOHTVlyhTr9vDwcA0bNszAZABcQfv27fXaa69p5cqVkv4oU0lJSRoxYgT/D9yDjh8/rhs3bhgdw+lUqFBB8fHx2RZ9Wr9+vapXr25QKtcSEBCQ7d8cPrCB2VFkHeT06dPq2bNntu1PPfWUpk2bZkAi18SnwACM9NZbb+n//u//VLJkSf3+++9q2rSpkpOT1aBBA02cONHoeEC+iIqK0gsvvKDr168rKytLO3bs0CeffKLJkyez0FY+ef/99+9oXExMjNLS0rhUJEyBIusgzZo10/fff6/KlSvbbN+6dauaNGliUCrXw6fAAIzk7++vr7/+WjExMUpISFBqaqrq1KmjiIgIo6MB+aZfv34qVKiQRo8erWvXrql79+4qU6aMZs2apa5duxodD3/RunVrxcfHcxUHmAJFNg99+eWX1q/bt2+vESNGKC4uTg899JCkP86R/fTTTzVhwgSjIrocPgUGYJQbN26oUKFCio+PV6NGjdSoUSOjIwGG6dGjh3r06KFr164pNTVVJUuWzDYmJiZG4eHhHA00EGvAwkxYtTgPubnd2dWMLBaLddEhON7HH3+s8ePHKzExUZJUpkwZTZgwQc8884zByQA4u4oVK+rzzz+3Lq6CexvXFDeWn58fRwMNxt8BmAnXkc1DmZmZd3SjxOavHj166PDhw0pNTVVycrJOnTpFiQWQL1599VW98sorunDhgtFRgHsex1YA2IOpxQZjqf/84+3tLW9vb6NjAHAhc+bM0ZEjR1SmTBmVL19ePj4+Nvt37dplUDLXdPXqVa1cuVJHjhxR6dKl1a1bNxUrVsy6/91331VgYKCBCQEAd4oiazCW+s97tWvXlsViuaOx/BIJwJE6duxodASXFhISoq1bt6po0aI6efKkHn74YV28eFH333+/EhMT9frrr2v79u2qUKGCJKl79+4GJwaMdae/PwH3AoosnA6/OAK4V4wbN87oCC7twIEDunnzpiRp1KhRKlOmjOLj4+Xv76/U1FQ9/vjjevXVV7Vs2TKDkwL3BqZ3w0wosnA6/OIIAPi72NhYzZ8/X/7+/pKkwoULa8KECVz+5R7C0UDHunz5spKTkyVJpUqVsv5d+Kvffvstv2MBuUaRhUv48ccftX//fkl/TDWrW7euwYkAOKuiRYvq0KFDKl68uIoUKXLbX85ZBMrx/vz5X79+XaVLl7bZV7ZsWZ09e9aIWMgBRwMdY+HChZo+fboOHjxos71q1aoaOnQoC2DCtCiycGqnTp1St27dFBMTo4CAAEnSpUuX1LBhQy1fvlz33XefsQEBOJ0ZM2bI19dXkjRz5kxjw0AtWrRQgQIFdOXKFR08eFA1atSw7jtx4oTNYk9wnJs3b2rz5s1KTExU9+7d5evrq19//VV+fn4qXLiwJI4GOsK0adM0fvx4vfjii2rVqpV1MbOUlBRt3LhRgwcP1sWLFzVs2DCDkwL24zqyBuN6XY712GOP6dKlS1q6dKmqVq0qSTp48KD69OkjPz8/rV+/3uCEAABHmTBhgs39hx56SK1atbLeHz58uE6dOqVPPvkkv6O5lBMnTuixxx5TUlKS0tLSdOjQIVWsWFGDBw9WWlqa5s+fb3REp1W+fHlNmzZNTz75ZI77V6xYoeHDhyspKSmfkwF3jyLrAOfOndPixYsVGxtrcy5Cw4YN1bt3b5UoUcI6dtmyZerQoUO2SzIgbxQqVEjbtm1T7dq1bbbHxcWpSZMmunbtmkHJADirK1eu3PFYPz8/ByYB7g0dO3aUr6+vFi1apGLFilk/wN+8ebP69++vw4cPGx3RaRUqVEi7du1S9erVc9y/b98+hYeH8/sQTImpxXls586datWqlby9vRUREaH7779f0h9TOGbPnq0pU6Zow4YNCg8Pl8RS/44WFBSU4+WNMjIyVKZMGQMSAXB2AQEBd7xoTUZGhoPTAMb7/vvvtW3bNnl4eNhsDw4O1i+//GJQKtfw4IMPasqUKVq0aJEKFLD9tT8jI0P//ve/9eCDDxqUDrg7FNk8NmjQID3xxBOaP39+tl9ksrKyNGDAAA0aNEixsbEGJXQt06ZN06BBgzR37lzrhwc//vijBg8erDfffNPgdACc0aZNm6xfHz9+XCNHjlTv3r3VoEEDSX+snrt06VJNnjzZqIhAvsrMzMzxQ5tTp05ZzyeHY8yZM0etWrVSqVKl9PDDD9ucI7tlyxZ5eHho48aNBqcEcoepxXmsUKFC2r17t6pVq5bj/gMHDqh27dr6/fff8zmZaypSpIiuXbummzdvWj+J/PPrv0/nZvVQAHmtRYsW6tevn7p162azfdmyZXrvvfe0efNmY4IB+ahLly7y9/fXe++9J19fX+3Zs0clSpRQhw4dVK5cOb3//vtGR3Rqv/32mz766CNt377d5pS3Bg0aqHv37pziANOiyOaxChUqaMKECerZs2eO+z/44AONHTtWx48fz99gLmrp0qV3PLZXr14OTALAFXl7eyshIUFVqlSx2X7o0CGFhYVxXhpcwqlTp9SqVStlZWXp8OHDCg8P1+HDh1W8eHFt2bJFJUuWNDoiABNianEeGzZsmJ599lnFxcWpRYsWNlM4oqOjtWDBAqa05iPKKQAjBQUFacGCBZo6darN9oULFyooKMigVED+uu+++5SQkKAVK1YoISFBqampeuaZZ9SjRw8VKlTI6Hgu7caNGzp9+rTKlStndBTAbhyRdYAVK1ZoxowZiouLs54T4u7urrp16yoqKuqWS6DDcc6cOaMzZ84oMzPTZnvNmjUNSgTAFaxbt06dO3dW5cqVVb9+fUnSjh07dPjwYX322Wdq06aNwQkBuLKEhATVqVOHhedgShRZB7px44bOnTsnSSpevLgKFixocCLXExcXp169emn//v36+x91i8XCP9wAHO7kyZOaN2+eDhw4IEmqXr26BgwYwBFZuIylS5eqePHiatu2rSTp5Zdf1nvvvaeQkBB98sknKl++vMEJXRdFFmZGkYVTq1WrlipVqqQRI0YoMDAw20rS/OcJAIBjVa1aVfPmzVPz5s0VGxurFi1aaObMmfrPf/6jAgUKaPXq1UZHdFp16tS57f7ff/9dhw4dosjClCiycGq+vr7avXu3KleubHQUAC5iz549qlGjhtzc3LRnz57bjuX0BrgCb29vHThwQOXKldOIESN0+vRpffDBB/r555/VrFkznT171uiITsvLy0tdu3ZVhQoVctx/+vRpLViwgCILU2KxJzi1Fi1aKCEhgSILIN+EhYUpOTlZJUuWVFhYmCwWS7ZTGyROb4DrKFy4sM6fP69y5cpp48aNioqKkvRHyeJyhI5Vo0YN1a9fXwMHDsxxf3x8vBYsWJDPqYC8QZGFU1u4cKF69eqln376STVq1Mh2nnL79u0NSgbAWR07dkwlSpSwfg24ukcffVT9+vVT7dq1dejQIesiZz///LOCg4ONDefkGjVqpIMHD95yv6+vrx5++OF8TATkHaYWw6l99dVXevrpp3XlypVs+zgaAgCA4126dEmjR4/WyZMnNXDgQD322GOSpHHjxsnDw0OvvvqqwQkBmBFFFk4tODhY//rXvzRmzBjrNX0BID8dPHhQb7/9tvbv3y/pj1WLBw0apKpVqxqcDABsPf/883rttddUvHhxo6MA/4giC6fm6+ur+Ph4VapUyegoAFzQZ599pq5duyo8PFwNGjSQJG3fvl07d+7U8uXL1blzZ4MTAo63ZcuW2+5nauu9w8/PT/Hx8apYsaLRUYB/RJGFU+vVq5eaNGmifv36GR0FgAuqVKmSevTooddee81m+7hx4/TRRx8pMTHRoGRA/nFzc8u27a+Xw+M0n3uHr6+vEhISKLIwBRZ7glO7//77NWrUKG3dulWhoaHZFnt68cUXDUoGwBWcPn1aPXv2zLb9qaee0rRp0wxIBOS/ixcv2ty/ceOGdu/erTFjxmjixIkGpQJgdhRZOLWFCxeqcOHC+u677/Tdd9/Z7LNYLBRZAA7VrFkzff/999kuAbZ161Y1adLEoFRA/vL398+27dFHH5WHh4eioqIUFxdnQCoAZkeRhVPj0hcA8tuXX35p/bp9+/YaMWKE4uLi9NBDD0n64xzZTz/9VBMmTDAqInBPCAwMvO2lYQDgdjhHFgCAPJTT+YA54RJgcBV79uyxuZ+VlaXTp09rypQpunnzprZu3WpQMvwd58jCTDgiC6cTFRWl119/XT4+PoqKirrt2OnTp+dTKgCuIjMz0+gIwD0lLCxMFotFfz928tBDD2nx4sUGpXIdN2/e1KRJk9S3b1/dd999tx371FNPyc/PL5+SAXeHI7JwOo888og+//xzBQQE6JFHHrnlOIvFom+//TYfkwFAzkJDQ7Vu3ToFBQUZHQXIcydOnLC57+bmphIlSsjLy8ugRK7H19dXe/fuVXBwsNFRgDxDkQUAwGBM5wP4QMeROnTooE6dOqlXr15GRwHyDFOL4VKuXLmib7/9VtWqVVO1atWMjgMAAP6/48eP68aNG0bHcEqtW7fWyJEjtXfvXtWtW1c+Pj42+9u3b29QMiD3OCILp/bkk0/q4YcfVmRkpH7//XfVqlVLx48fV1ZWlpYvX67OnTsbHREAOCILiL8HjnS7RehYeA5mdWdLKwImtWXLFuu1Gj///HNlZWXp0qVLmj17tt544w2D0wEAADheZmbmLW+UWJgVRRZO7fLlyypatKgkaf369ercubO8vb3Vtm1bHT582OB0AAAA+ev69etGRwDyBEUWTi0oKEixsbG6evWq1q9fr5YtW0qSLl68yGqJAPIdZ/MAMEJGRoZef/11lS1bVoULF9bRo0clSWPGjNGiRYsMTgfkDkUWTm3IkCHq0aOH7rvvPpUpU0bNmjWT9MeU49DQUGPDAXA5np6e2r9/f7bt7777rgIDAw1IBMAVTJw4UUuWLNHUqVPl4eFh3V6jRg0tXLjQwGRA7rHYE5xeXFyckpKS9Oijj6pw4cKSpLVr1yogIECNGjUyOB0AZxQVFZXj9lmzZumpp55SsWLFJEnTp0/Pz1jAPW3ZsmXq0KFDthV1cfcqV66sd999Vy1atLBZVOvAgQNq0KCBLl68aHREwG4UWUCSn5+f4uPjWSkRQJ5wc3NTrVq1FBAQYLP9u+++U3h4uHx8fGSxWPTtt98aExDIZ9HR0ZoxY4Z1RkL16tU1ZMgQRUREGJzMNRQqVEgHDhxQ+fLlbYrsvn37VK9ePaWmphodEbAbU4sBcd4agLw1adIkXb58WWPGjNGmTZusN3d3dy1ZskSbNm2ixMJlvPPOO3rsscfk6+urwYMHa/DgwfLz81ObNm00d+5co+O5hJCQEH3//ffZtq9atUq1a9c2IBFw9woYHQAAAGczcuRItWjRQk899ZTatWunyZMnq2DBgkbHAgwxadIkzZgxQ5GRkdZtL774oho1aqRJkybphRdeMDCdaxg7dqx69eqlX375RZmZmVq9erUOHjyoDz74QP/5z3+MjgfkCkdkAQBwgAcffFBxcXE6e/aswsPD9dNPP8lisRgdC8h3ly5d0mOPPZZte8uWLXX58mUDErmeDh066KuvvtI333wjHx8fjR07Vvv379dXX32lRx991Oh4QK5wRBYAAAcpXLiwli5dquXLlysiIkIZGRlGRwLyXfv27fX5559r+PDhNtu/+OIL/etf/zIoletp0qSJvv76a6NjAHmGIgtIHCUB4FBdu3ZV48aNFRcXp/LlyxsdB3C42bNnW78OCQnRxIkTtXnzZjVo0ECStH37dsXExGjo0KFGRQRgcqxaDEg2K/gBAIC7U6FChTsaZ7FYdPToUQencU1FihS54w/qL1y44OA0QN7jiCxc0smTJzVu3DgtXrxYkvTf//5XZcuWNTgVAADO4dixY0ZHcHkzZ860fn3+/Hm98cYbatWqlfWoeGxsrDZs2KAxY8YYlBC4OxyRhUtKSEhQnTp1OF8NAAA4vc6dO+uRRx6xWTlakubMmaNvvvlGa9asMSYYcBcosnBKX3755W33Hz16VEOHDqXIAgDgYH379r3t/j9nR8FxChcurPj4eFWuXNlm+5EjRxQWFqbU1FSDkgG5x9RiOKWOHTvKYrHodp/TsMATAACOd/HiRZv7N27c0E8//aRLly6pefPmBqVyLcWKFdMXX3yRbXGtL774QsWKFTMoFXB3KLJwSqVLl9Y777yjDh065Lg/Pj5edevWzedUAAC4ns8//zzbtszMTA0cOFCVKlUyIJHrmTBhgvr166fNmzerfv36kqQffvhB69ev14IFCwxOB+SOm9EBAEeoW7eu4uLibrn/n47WAgAAx3Fzc1NUVJRmzJhhdBSX0Lt3b8XExMjPz0+rV6/W6tWr5efnp61bt6p3795GxwNyhSOycErDhw/X1atXb7m/cuXK2rRpUz4mAgAAf5WYmKibN28aHcNl1K9fXx9//LHRMYA8w2JPAAAAcJioqCib+1lZWTp9+rTWrl2rXr16ac6cOQYlcy0ZGRlas2aN9u/fL0l64IEH1L59e7m7uxucDMgdiiwAAAAc5pFHHrG57+bmphIlSqh58+bq27evChRggqCjHTlyRG3bttWpU6dUtWpVSdLBgwcVFBSktWvXcq4yTIkiCwAAADixNm3aKCsrSx9//LGKFi0qSTp//ryeeuopubm5ae3atQYnBOxHkQUAAACcmI+Pj7Zv367Q0FCb7QkJCWrUqBHXkYUpsWoxAAAAHCYlJUVPP/20ypQpowIFCsjd3d3mBsfz9PTUb7/9lm17amqqPDw8DEgE3D1OSgAAAIDD9O7dW0lJSRozZoxKly4ti8VidCSX869//UvPPvusFi1apHr16kn64zqyAwYMUPv27Q1OB+QOU4sBAADgML6+vvr+++8VFhZmdBSXdenSJfXq1UtfffWVChYsKEm6efOm2rdvryVLlsjf39/ghID9OCILAAAAhwkKChLHTYwVEBCgL774QkeOHLFefqd69eqqXLmywcmA3OOILAAAABxm48aNeuutt/Tuu+8qODjY6DgAnARFFgAAAHmqSJEiNufCXr16VTdv3pS3t7d1auufLly4kN/xXE7nzp1Vr149jRgxwmb71KlTtXPnTn366acGJQNyjyILAACAPLV06dI7HturVy8HJoEklShRQt9++222y+/s3btXERERSklJMSgZkHucIwsAAIA8lZtyOmXKFA0YMEABAQF5H8jF3eoyOwULFtSVK1cMSATcPa4jCwAAAMNNmjSJacYOEhoaqhUrVmTbvnz5coWEhBiQCLh7HJEFAACA4TjbzXHGjBmjTp06KTExUc2bN5ckRUdH65NPPuH8WJgWRRYAAABwYu3atdOaNWs0adIkrVq1SoUKFVLNmjX1zTffqGnTpkbHA3KFxZ4AAABgOF9fXyUkJKhixYpGRwFgAhyRBQAAAFxAenq6zpw5o8zMTJvt5cqVMygRkHsUWQAAAMCJHT58WH379tW2bdtstmdlZclisSgjI8OgZEDuUWQBAACQp6KiovT666/Lx8dHW7ZsUcOGDVWgwO1/7WzSpIkKFSqUTwldS+/evVWgQAH95z//UenSpWWxWIyOBNw1zpEFAABAnipYsKBOnTqlwMBAubu76/Tp0ypZsqTRsVyWj4+P4uLiVK1aNaOjAHmGI7IAAADIU8HBwZo9e7ZatmyprKwsxcbGqkiRIjmOffjhh/M5nesJCQnRuXPnjI4B5CmOyAIAACBPrVmzRgMGDNCZM2dksVhueY1Yzs/MH99++61Gjx6tSZMmKTQ0VAULFrTZ7+fnZ1AyIPcosgAAAHCI1NRU+fn56eDBg7ecWuzv75/PqVyPm5ubJGU7N5bFnmBmTC0GAACAQxQuXFibNm1ShQoV/nGxJzjOpk2bjI4A5DmOyAIAAMBhbrXY0/nz51WyZEmOBgLIFTejAwAAAMB53eqYSVpamjw8PPI5jev6/vvv9dRTT6lhw4b65ZdfJEkffvihtm7danAyIHeY4wEAAIA8N3v2bEl/nJe5cOFCFS5c2LovIyNDW7Zs4XIw+eSzzz7T008/rR49emjXrl1KS0uTJF2+fFmTJk3SunXrDE4I2I+pxQAAAMhzFSpUkCSdOHFC9913n9zd3a37PDw8FBwcrNdee03169c3KqLLqF27tl566SX17NlTvr6+SkhIUMWKFbV79261bt1aycnJRkcE7MYRWQAAAOS5Y8eOSZIeeeQRrV69+pbXkYXjHTx4MMfr9fr7++vSpUv5HwjIA5wjCwAAAIfZtGnTHZVYPz8/HT16NB8SuZ5SpUrpyJEj2bZv3bpVFStWNCARcPcosgAAADAcZ7s5Tv/+/TV48GD98MMPslgs+vXXX/Xxxx9r2LBhGjhwoNHxgFxhajEAAADgxEaOHKnMzEy1aNFC165d08MPPyxPT08NGzZMgwYNMjoekCss9gQAAADD/XURIjhGenq6jhw5otTUVIWEhNisJC1Jp06dUpkyZeTmxqRN3Ps4IgsAAAC4AA8PD4WEhNxyf0hIiOLj4/kwAabAxy0AAAAwnMViMTqCy2OiJsyEIgsAAADDUaIA2IMiCwAAAMP997//VdmyZY2OAcAkKLIAAADIc7t27dKxY8es9z/88EM1atRIQUFBaty4sZYvX24zvnHjxvL09MzvmABMiiILAACAPNenTx8lJiZKkhYuXKjnnntO4eHhevXVV/Xggw+qf//+Wrx4scEp8VecpwwzYdViAAAA5LnDhw+rSpUqkqR33nlHs2bNUv/+/a37H3zwQU2cOFF9+/Y1KiL+hvOUYSYckQUAAECe8/b21rlz5yRJv/zyi+rVq2ezv379+jZTj+F4R44c0YYNG/T7779Lyl5c9+3bp/LlyxsRDbAbRRYAAAB5rnXr1po3b54kqWnTplq1apXN/pUrV6py5cpGRHM558+fV0REhO6//361adNGp0+fliQ988wzGjp0qHVcUFCQ3N3djYoJ2MWSxRwCAAAA5LFff/1VjRo1Urly5RQeHq558+apbt26ql69ug4ePKjt27fr888/V5s2bYyO6vR69uypM2fOaOHChapevboSEhJUsWJFbdiwQVFRUfr555+NjgjYjXNkAQAAkOfKlCmj3bt3a8qUKfrqq6+UlZWlHTt26OTJk2rUqJFiYmIUHh5udEyXsHHjRm3YsEH33XefzfYqVaroxIkTBqUC7g5FFgAAAA4REBCgKVOmaMqUKUZHcWlXr16Vt7d3tu0XLlzgkkcwLc6RBQAAAJxYkyZN9MEHH1jvWywWZWZmaurUqXrkkUcMTAbkHufIAgAAAE7sp59+UosWLVSnTh19++23at++vX7++WdduHBBMTExqlSpktERAbtRZAEAAAAnd/nyZc2ZM0cJCQlKTU1VnTp19MILL6h06dJGRwNyhSILAAAAADAVzpEFAAAAnNj69eu1detW6/25c+cqLCxM3bt318WLFw1MBuQeRRYAAABwYsOHD9eVK1ckSXv37lVUVJTatGmjY8eOKSoqyuB0QO5w+R0AAADAiR07dkwhISGSpM8++0zt2rXTpEmTtGvXLrVp08bgdEDucEQWAAAAcGIeHh66du2aJOmbb75Ry5YtJUlFixa1HqkFzIYjsgAAAIATa9y4saKiotSoUSPt2LFDK1askCQdOnRI9913n8HpgNzhiCwAAADgxObMmaMCBQpo1apVmjdvnsqWLStJ+u9//6vHHnvM4HRA7nD5HQAAAACAqTC1GAAAAHBiSUlJt91frly5fEoC5B2OyAIAAABOzM3NTRaL5Zb7MzIy8jENkDc4IgsAAAA4sd27d9vcv3Hjhnbv3q3p06dr4sSJBqUC7g5HZAEAAAAXtHbtWk2bNk2bN282OgpgN1YtBgAAAFxQ1apVtXPnTqNjALnC1GIAAADAiV25csXmflZWlk6fPq3x48erSpUqBqUC7g5FFgAAAHBiAQEB2RZ7ysrKUlBQkJYvX25QKuDucI4sAAAA4MS+++47m/tubm4qUaKEKleurAIFOK4Fc6LIAgAAAFDbtm21cOFClS5d2ugowD9isScAAAAA2rJli37//XejYwB3hCILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAvfLKKypatKjRMYA7wnVkAQAAACd38OBBvf3229q/f78kqXr16ho0aJCqVq1qcDIgdzgiCwAAADixzz77TDVq1FBcXJxq1aqlWrVqadeuXapRo4Y+++wzo+MBucIRWQAAAMCJVapUST169NBrr71ms33cuHH66KOPlJiYaFAyIPcosgAAAIAT8/b21p49e1S5cmWb7YcPH1atWrV07do1g5IBucfUYgAAAMCJNWvWTN9//3227Vu3blWTJk0MSATcvQJGBwAAAACQt7788kvr1+3bt9eIESMUFxenhx56SJK0fft2ffrpp5owYYJREYG7wtRiAAAAwMm4ud3ZxEuLxaKMjAwHpwHyHkUWAAAAAGAqnCMLAAAAADAVzpEFAAAAnNjfL7vzd2PHjs2nJEDeYWoxAAAA4MRq165tc//GjRs6duyYChQooEqVKmnXrl0GJQNyjyOyAAAAgBPbvXt3tm1XrlxR79699fjjjxuQCLh7HJEFAAAAXNDevXvVrl07HT9+3OgogN1Y7AkAAABwQZcvX9bly5eNjgHkClOLAQAAACc2e/Zsm/tZWVk6ffq0PvzwQ7Vu3dqgVMDdYWoxAAAA4MQqVKhgc9/NzU0lSpRQ8+bNNWrUKPn6+hqUDMg9iiwAAAAAwFQ4RxYAAAAAYCqcIwsAAAA4satXr2rKlCmKjo7WmTNnlJmZabP/6NGjBiUDco8iCwAAADixfv366bvvvtPTTz+t0qVLy2KxGB0JuGucIwsAAAA4sYCAAK1du1aNGjUyOgqQZzhHFgAAAHBiRYoUUdGiRY2OAeQpiiwAAADgxF5//XWNHTtW165dMzoKkGeYWgwAAAA4mdq1a9ucC3vkyBFlZWUpODhYBQsWtBm7a9eu/I4H3DUWewIAAACcTMeOHY2OADgUR2QBAAAA6JNPPlH79u3l4+NjdBTgH1FkAQAAAMjPz0/x8fGqWLGi0VGAf8RiTwAAAADE8S2YCUUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAcDKzZ8/W9evXJUlJSUl3tJBT+fLlVbBgQUdHA/IEl98BAAAAnEyBAgX066+/qmTJknJ3d9fp06dVsmRJo2MBeaaA0QEAAAAA5K0yZcros88+U5s2/6+d+9eFZo/jOP6dFVEqCBprUS9XoREJpU6h3itQ6Um0FELiCkRDVqV0CxLLIkoJzYYxpzg5kvOcRyLP7NnJTF6vav9M8Sn3nd/MLkeWZfHw8PB1Qvurer0+4HWQnxNZAAComIODg2i1WvHx8fHtNVmWRZIkkabpAJdBfwhZAACooNfX17i7u4uFhYVot9sxNjb22+sWFxcHvAzyE7IAAFBhx8fHsb6+HiMjI0VPgb7xr8UAAFBh29vb8fb29p/PX15eYm5uroBFkJ+QBQCACut0Or99DrbX68Xj42MBiyA//1oMAAAVdHp6+vX6/Pw8RkdHv96naRqXl5fRaDQKWAb5eUYWAAAqqFb7++bLJEni15/8w8PD0Wg0Ynd3N1ZWVoqYB7kIWQAAqLDZ2dm4vr6O8fHxoqdA33hGFgAAKuz29vZHEdtsNqPb7Q5gEeQnZAEAgOh0OvH+/l70DPgRIQsAAECpCFkAAABKRcgCAABQKkIWAACAUhGyAAAAlIqQBQCACmq1WnF1dfXj6/f392NycvJ/XAT9k2RZlhU9AgAA6K9arRZJksT8/Hxsbm7GxsZGTE1NFT0L+sKJLAAAVNTFxUUsLy/Hzs5O1Ov1WF1djbOzs/j8/Cx6GuQiZAEAoKKazWbs7e3F09NTnJycRK/Xi7W1tZieno6tra24ubkpeiL8EbcWAwBABdVqtXh+fo6JiYl/fX5/fx+Hh4dxdHQU3W430jQtaCH8OSELAAAV9F3I/iPLsmi327G0tDTgZZCfW4sBAKCCZmZmYmho6NvvkyQRsZSWE1kAAABKxYksAAAApSJkAQAAKBUhCwAAQKkIWQAAAEpFyAIAAFAqQhYAAIBSEbIAAACUipAFAACgVP4CV3n/HC/fivMAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Sort model results by F1-scores\n",
        "all_model_results.sort_values(\"f1\", ascending=False)[\"f1\"].plot(kind=\"bar\", figsize=(10,7));"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 825
        },
        "id": "sgnlAsBhMmUs",
        "outputId": "4e53177b-a3b9-48ae-d5d2-ca84881739f7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzoAAAMoCAYAAAAHr5UkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABn10lEQVR4nO3de3zO9eP/8ee1sTG2Oc9pzCksZ4scO1iEHMq3HKOFoia1COUc5qNiRJRDdBASUoQsEk20sZTjnEZsyCmTjW2/P/y66moj067rXa/rcb/drttte73f1/bkKtvzer9er7ctMzMzUwAAAABgEA+rAwAAAABAbqPoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDh5rA5wKzIyMnTixAn5+vrKZrNZHQcAAACARTIzM/Xrr7+qdOnS8vC48XWb/0TROXHihAIDA62OAQAAAOBf4tixYypbtuwNj/8nio6vr6+k638YPz8/i9MAAAAAsMrFixcVGBho7wg38p8oOr9PV/Pz86PoAAAAAPjbJS1sRgAAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjHNbRWfGjBkKCgpSvnz51LBhQ23btu2m50dFRalq1arKnz+/AgMD9cILL+jKlSu3FRgAAAAA/k6Oi87ixYsVERGhUaNGKS4uTrVr11arVq106tSpbM9fuHChhg4dqlGjRmnPnj2aO3euFi9erJdffvkfhwcAAACA7NgyMzMzc/KEhg0b6q677tL06dMlSRkZGQoMDNSAAQM0dOjQLOeHh4drz549io6Oto+9+OKL+u6777R58+Zb+p4XL16Uv7+/Lly4ID8/v5zEvWVBQ1c55eu6ypGJba2OAAAAADjdrXaDHF3RSUtLU2xsrEJDQ//4Ah4eCg0NVUxMTLbPady4sWJjY+3T2w4dOqTVq1erTZs2N/w+qampunjxosMDAAAAAG5VnpycfObMGaWnpysgIMBhPCAgQHv37s32Od26ddOZM2fUtGlTZWZm6tq1a+rXr99Np65FRkZqzJgxOYkGAAAAAHZO33Vt48aNmjBhgt566y3FxcVp2bJlWrVqlV599dUbPmfYsGG6cOGC/XHs2DFnxwQAAABgkBxd0SlWrJg8PT2VnJzsMJ6cnKySJUtm+5wRI0bo8ccfV58+fSRJNWvWVEpKip566im98sor8vDI2rW8vb3l7e2dk2gAAAAAYJejKzpeXl6qX7++w8YCGRkZio6OVqNGjbJ9zuXLl7OUGU9PT0lSDvdBAAAAAIBbkqMrOpIUERGhXr16KSQkRA0aNFBUVJRSUlIUFhYmSerZs6fKlCmjyMhISVK7du00efJk1a1bVw0bNlRCQoJGjBihdu3a2QsPAAAAAOSmHBedzp076/Tp0xo5cqSSkpJUp04drVmzxr5BQWJiosMVnOHDh8tms2n48OH6+eefVbx4cbVr107jx4/PvT8FjMAW3wAAAMgtOb6PjhW4j87fM+GXbF4DAAAA/B2n3EcHAAAAAP4LKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABgnx9tLAzAXO98BAABTUHQA4F+EsgkAQO5g6hoAAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOHmsDgAAwL9F0NBVVkf4x45MbGt1BAD4V+CKDgAAAADjUHQAAAAAGIepawAA4F+D6YMAcgtFBwAAAHaUTZiCqWsAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOOy6BgAAAPyLsPNd7uCKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgnNsqOjNmzFBQUJDy5cunhg0batu2bTc8995775XNZsvyaNvW+psIAQAAADBTjovO4sWLFRERoVGjRikuLk61a9dWq1atdOrUqWzPX7ZsmU6ePGl//Pjjj/L09NSjjz76j8MDAAAAQHZyXHQmT56svn37KiwsTMHBwZo1a5Z8fHw0b968bM8vUqSISpYsaX98+eWX8vHxoegAAAAAcJocFZ20tDTFxsYqNDT0jy/g4aHQ0FDFxMTc0teYO3euunTpogIFCtzwnNTUVF28eNHhAQAAAAC3KkdF58yZM0pPT1dAQIDDeEBAgJKSkv72+du2bdOPP/6oPn363PS8yMhI+fv72x+BgYE5iQkAAADAzbl017W5c+eqZs2aatCgwU3PGzZsmC5cuGB/HDt2zEUJAQAAAJggT05OLlasmDw9PZWcnOwwnpycrJIlS970uSkpKVq0aJHGjh37t9/H29tb3t7eOYkGAAAAAHY5uqLj5eWl+vXrKzo62j6WkZGh6OhoNWrU6KbP/fjjj5WamqoePXrcXlIAAAAAuEU5uqIjSREREerVq5dCQkLUoEEDRUVFKSUlRWFhYZKknj17qkyZMoqMjHR43ty5c9WxY0cVLVo0d5IDAAAAwA3kuOh07txZp0+f1siRI5WUlKQ6depozZo19g0KEhMT5eHheKFo37592rx5s9atW5c7qQEAAADgJnJcdCQpPDxc4eHh2R7buHFjlrGqVasqMzPzdr4VAAAAAOSYS3ddAwAAAABXoOgAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGCc2yo6M2bMUFBQkPLly6eGDRtq27ZtNz3//PnzevbZZ1WqVCl5e3vrjjvu0OrVq28rMAAAAAD8nTw5fcLixYsVERGhWbNmqWHDhoqKilKrVq20b98+lShRIsv5aWlpeuCBB1SiRAktXbpUZcqU0dGjR1WoUKHcyA8AAAAAWeS46EyePFl9+/ZVWFiYJGnWrFlatWqV5s2bp6FDh2Y5f968eTp79qy+/fZb5c2bV5IUFBT0z1IDAAAAwE3kaOpaWlqaYmNjFRoa+scX8PBQaGioYmJisn3OypUr1ahRIz377LMKCAhQjRo1NGHCBKWnp9/w+6SmpurixYsODwAAAAC4VTkqOmfOnFF6eroCAgIcxgMCApSUlJTtcw4dOqSlS5cqPT1dq1ev1ogRI/TGG29o3LhxN/w+kZGR8vf3tz8CAwNzEhMAAACAm3P6rmsZGRkqUaKE3nnnHdWvX1+dO3fWK6+8olmzZt3wOcOGDdOFCxfsj2PHjjk7JgAAAACD5GiNTrFixeTp6ank5GSH8eTkZJUsWTLb55QqVUp58+aVp6enfax69epKSkpSWlqavLy8sjzH29tb3t7eOYkGAAAAAHY5uqLj5eWl+vXrKzo62j6WkZGh6OhoNWrUKNvnNGnSRAkJCcrIyLCP7d+/X6VKlcq25AAAAADAP5XjqWsRERGaPXu2FixYoD179qh///5KSUmx78LWs2dPDRs2zH5+//79dfbsWQ0cOFD79+/XqlWrNGHCBD377LO596cAAAAAgD/J8fbSnTt31unTpzVy5EglJSWpTp06WrNmjX2DgsTERHl4/NGfAgMDtXbtWr3wwguqVauWypQpo4EDB2rIkCG596cAAAAAgD/JcdGRpPDwcIWHh2d7bOPGjVnGGjVqpK1bt97OtwIAAACAHHP6rmsAAAAA4GoUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxbqvozJgxQ0FBQcqXL58aNmyobdu23fDc+fPny2azOTzy5ct324EBAAAA4O/kuOgsXrxYERERGjVqlOLi4lS7dm21atVKp06duuFz/Pz8dPLkSfvj6NGj/yg0AAAAANxMjovO5MmT1bdvX4WFhSk4OFizZs2Sj4+P5s2bd8Pn2Gw2lSxZ0v4ICAj4R6EBAAAA4GZyVHTS0tIUGxur0NDQP76Ah4dCQ0MVExNzw+ddunRJ5cuXV2BgoDp06KCffvrp9hMDAAAAwN/IUdE5c+aM0tPTs1yRCQgIUFJSUrbPqVq1qubNm6dPP/1UH3zwgTIyMtS4cWMdP378ht8nNTVVFy9edHgAAAAAwK1y+q5rjRo1Us+ePVWnTh3dc889WrZsmYoXL6633377hs+JjIyUv7+//REYGOjsmAAAAAAMkqOiU6xYMXl6eio5OdlhPDk5WSVLlrylr5E3b17VrVtXCQkJNzxn2LBhunDhgv1x7NixnMQEAAAA4OZyVHS8vLxUv359RUdH28cyMjIUHR2tRo0a3dLXSE9P165du1SqVKkbnuPt7S0/Pz+HBwAAAADcqjw5fUJERIR69eqlkJAQNWjQQFFRUUpJSVFYWJgkqWfPnipTpowiIyMlSWPHjtXdd9+typUr6/z583rttdd09OhR9enTJ3f/JAAAAADw/+W46HTu3FmnT5/WyJEjlZSUpDp16mjNmjX2DQoSExPl4fHHhaJz586pb9++SkpKUuHChVW/fn19++23Cg4Ozr0/BQAAAAD8SY6LjiSFh4crPDw822MbN250+HzKlCmaMmXK7XwbAAAAALgtTt91DQAAAABcjaIDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxbqvozJgxQ0FBQcqXL58aNmyobdu23dLzFi1aJJvNpo4dO97OtwUAAACAW5LjorN48WJFRERo1KhRiouLU+3atdWqVSudOnXqps87cuSIBg0apGbNmt12WAAAAAC4FTkuOpMnT1bfvn0VFham4OBgzZo1Sz4+Ppo3b94Nn5Oenq7u3btrzJgxqlix4j8KDAAAAAB/J0dFJy0tTbGxsQoNDf3jC3h4KDQ0VDExMTd83tixY1WiRAn17t379pMCAAAAwC3Kk5OTz5w5o/T0dAUEBDiMBwQEaO/evdk+Z/PmzZo7d6527tx5y98nNTVVqamp9s8vXryYk5gAAAAA3JxTd1379ddf9fjjj2v27NkqVqzYLT8vMjJS/v7+9kdgYKATUwIAAAAwTY6u6BQrVkyenp5KTk52GE9OTlbJkiWznH/w4EEdOXJE7dq1s49lZGRc/8Z58mjfvn2qVKlSlucNGzZMERER9s8vXrxI2QEAAABwy3JUdLy8vFS/fn1FR0fbt4jOyMhQdHS0wsPDs5xfrVo17dq1y2Fs+PDh+vXXXzV16tQblhdvb295e3vnJBoAAAAA2OWo6EhSRESEevXqpZCQEDVo0EBRUVFKSUlRWFiYJKlnz54qU6aMIiMjlS9fPtWoUcPh+YUKFZKkLOMAAAAAkFtyXHQ6d+6s06dPa+TIkUpKSlKdOnW0Zs0a+wYFiYmJ8vBw6tIfAAAAALipHBcdSQoPD892qpokbdy48abPnT9//u18SwAAAAC4ZVx6AQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMa5raIzY8YMBQUFKV++fGrYsKG2bdt2w3OXLVumkJAQFSpUSAUKFFCdOnX0/vvv33ZgAAAAAPg7OS46ixcvVkREhEaNGqW4uDjVrl1brVq10qlTp7I9v0iRInrllVcUExOjH374QWFhYQoLC9PatWv/cXgAAAAAyE6Oi87kyZPVt29fhYWFKTg4WLNmzZKPj4/mzZuX7fn33nuvHn74YVWvXl2VKlXSwIEDVatWLW3evPkfhwcAAACA7OSo6KSlpSk2NlahoaF/fAEPD4WGhiomJuZvn5+Zmano6Gjt27dPzZs3v+F5qampunjxosMDAAAAAG5VjorOmTNnlJ6eroCAAIfxgIAAJSUl3fB5Fy5cUMGCBeXl5aW2bdvqzTff1AMPPHDD8yMjI+Xv729/BAYG5iQmAAAAADfnkl3XfH19tXPnTm3fvl3jx49XRESENm7ceMPzhw0bpgsXLtgfx44dc0VMAAAAAIbIk5OTixUrJk9PTyUnJzuMJycnq2TJkjd8noeHhypXrixJqlOnjvbs2aPIyEjde++92Z7v7e0tb2/vnEQDAAAAALscXdHx8vJS/fr1FR0dbR/LyMhQdHS0GjVqdMtfJyMjQ6mpqTn51gAAAABwy3J0RUeSIiIi1KtXL4WEhKhBgwaKiopSSkqKwsLCJEk9e/ZUmTJlFBkZKen6epuQkBBVqlRJqampWr16td5//33NnDkzd/8kAAAAAPD/5bjodO7cWadPn9bIkSOVlJSkOnXqaM2aNfYNChITE+Xh8ceFopSUFD3zzDM6fvy48ufPr2rVqumDDz5Q586dc+9PAQAAAAB/kuOiI0nh4eEKDw/P9thfNxkYN26cxo0bdzvfBgAAAABui0t2XQMAAAAAV6LoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgnNsqOjNmzFBQUJDy5cunhg0batu2bTc8d/bs2WrWrJkKFy6swoULKzQ09KbnAwAAAMA/leOis3jxYkVERGjUqFGKi4tT7dq11apVK506dSrb8zdu3KiuXbtqw4YNiomJUWBgoFq2bKmff/75H4cHAAAAgOzkuOhMnjxZffv2VVhYmIKDgzVr1iz5+Pho3rx52Z7/4Ycf6plnnlGdOnVUrVo1zZkzRxkZGYqOjv7H4QEAAAAgOzkqOmlpaYqNjVVoaOgfX8DDQ6GhoYqJibmlr3H58mVdvXpVRYoUueE5qampunjxosMDAAAAAG5VjorOmTNnlJ6eroCAAIfxgIAAJSUl3dLXGDJkiEqXLu1Qlv4qMjJS/v7+9kdgYGBOYgIAAABwcy7ddW3ixIlatGiRli9frnz58t3wvGHDhunChQv2x7Fjx1yYEgAAAMB/XZ6cnFysWDF5enoqOTnZYTw5OVklS5a86XNff/11TZw4UevXr1etWrVueq63t7e8vb1zEg0AAAAA7HJ0RcfLy0v169d32Ejg940FGjVqdMPnTZo0Sa+++qrWrFmjkJCQ208LAAAAALcgR1d0JCkiIkK9evVSSEiIGjRooKioKKWkpCgsLEyS1LNnT5UpU0aRkZGSpP/9738aOXKkFi5cqKCgIPtanoIFC6pgwYK5+EcBAAAAgOtyXHQ6d+6s06dPa+TIkUpKSlKdOnW0Zs0a+wYFiYmJ8vD440LRzJkzlZaWpv/7v/9z+DqjRo3S6NGj/1l6AAAAAMhGjouOJIWHhys8PDzbYxs3bnT4/MiRI7fzLQAAAADgtrl01zUAAAAAcAWKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxrmtojNjxgwFBQUpX758atiwobZt23bDc3/66Sd16tRJQUFBstlsioqKut2sAAAAAHBLclx0Fi9erIiICI0aNUpxcXGqXbu2WrVqpVOnTmV7/uXLl1WxYkVNnDhRJUuW/MeBAQAAAODv5LjoTJ48WX379lVYWJiCg4M1a9Ys+fj4aN68edmef9ddd+m1115Tly5d5O3t/Y8DAwAAAMDfyVHRSUtLU2xsrEJDQ//4Ah4eCg0NVUxMTK6FSk1N1cWLFx0eAAAAAHCrclR0zpw5o/T0dAUEBDiMBwQEKCkpKddCRUZGyt/f3/4IDAzMta8NAAAAwHz/yl3Xhg0bpgsXLtgfx44dszoSAAAAgP+QPDk5uVixYvL09FRycrLDeHJycq5uNODt7c16HgAAAAC3LUdXdLy8vFS/fn1FR0fbxzIyMhQdHa1GjRrlejgAAAAAuB05uqIjSREREerVq5dCQkLUoEEDRUVFKSUlRWFhYZKknj17qkyZMoqMjJR0fQOD3bt32z/++eeftXPnThUsWFCVK1fOxT8KAAAAAFyX46LTuXNnnT59WiNHjlRSUpLq1KmjNWvW2DcoSExMlIfHHxeKTpw4obp169o/f/311/X666/rnnvu0caNG//5nwAAAAAA/iLHRUeSwsPDFR4enu2xv5aXoKAgZWZm3s63AQAAAIDb8q/cdQ0AAAAA/gmKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4t1V0ZsyYoaCgIOXLl08NGzbUtm3bbnr+xx9/rGrVqilfvnyqWbOmVq9efVthAQAAAOBW5LjoLF68WBERERo1apTi4uJUu3ZttWrVSqdOncr2/G+//VZdu3ZV7969tWPHDnXs2FEdO3bUjz/++I/DAwAAAEB2clx0Jk+erL59+yosLEzBwcGaNWuWfHx8NG/evGzPnzp1qh588EENHjxY1atX16uvvqp69epp+vTp/zg8AAAAAGQnR0UnLS1NsbGxCg0N/eMLeHgoNDRUMTEx2T4nJibG4XxJatWq1Q3PBwAAAIB/Kk9OTj5z5ozS09MVEBDgMB4QEKC9e/dm+5ykpKRsz09KSrrh90lNTVVqaqr98wsXLkiSLl68mJO4OZKRetlpX9sVnPl34yq8BtbjNbAer4G1/ut//xKvwb8Br4H1eA2s58zX4PevnZmZedPzclR0XCUyMlJjxozJMh4YGGhBmv8G/yirE4DXwHq8BtbjNbAer4H1eA2sx2tgPVe8Br/++qv8/f1veDxHRadYsWLy9PRUcnKyw3hycrJKliyZ7XNKliyZo/MladiwYYqIiLB/npGRobNnz6po0aKy2Ww5ifyvcPHiRQUGBurYsWPy8/OzOo5b4jWwHq+B9XgNrMdrYD1eA2vx9289E16DzMxM/frrrypduvRNz8tR0fHy8lL9+vUVHR2tjh07SrpeQqKjoxUeHp7tcxo1aqTo6Gg9//zz9rEvv/xSjRo1uuH38fb2lre3t8NYoUKFchL1X8nPz+8/+x+UKXgNrMdrYD1eA+vxGliP18Ba/P1b77/+GtzsSs7vcjx1LSIiQr169VJISIgaNGigqKgopaSkKCwsTJLUs2dPlSlTRpGRkZKkgQMH6p577tEbb7yhtm3batGiRfr+++/1zjvv5PRbAwAAAMAtyXHR6dy5s06fPq2RI0cqKSlJderU0Zo1a+wbDiQmJsrD44/N3Bo3bqyFCxdq+PDhevnll1WlShWtWLFCNWrUyL0/BQAAAAD8yW1tRhAeHn7DqWobN27MMvboo4/q0UcfvZ1vZQRvb2+NGjUqy3Q8uA6vgfV4DazHa2A9XgPr8RpYi79/67nTa2DL/Lt92QAAAADgPyZHNwwFAAAAgP8Cig4AAAAA41B0AAAAABiHogMAAADAOBQdJ7h69aqefPJJHT582OooAAAAgFti1zUn8ff3186dO1WhQgWro7i9tLQ0HT58WJUqVVKePLe1ozoA/CPbt2/Xhg0bdOrUKWVkZDgcmzx5skWpAMBs/NbnJB07dtSKFSv0wgsvWB3FbV2+fFkDBgzQggULJEn79+9XxYoVNWDAAJUpU0ZDhw61OKH7OHjwoN59910dPHhQU6dOVYkSJfTFF1+oXLlyuvPOO62OZzRPT0+dPHlSJUqUcBj/5ZdfVKJECaWnp1uUzH1MmDBBw4cPV9WqVRUQECCbzWY/9uePkXseeeSRWz532bJlTkwCWGflypW3fG779u2dmMQ6FB0nqVKlisaOHastW7aofv36KlCggMPx5557zqJk7mPYsGGKj4/Xxo0b9eCDD9rHQ0NDNXr0aIqOi3z99ddq3bq1mjRpok2bNmn8+PEqUaKE4uPjNXfuXC1dutTqiEa70UX71NRUeXl5uTiNe5o6darmzZunJ554wuoobsPf39/qCPiTChUq3LTUHzp0yIVp3EfHjh0dPrfZbA4/E/78mpj6phdFx0nmzp2rQoUKKTY2VrGxsQ7HbDYbRccFVqxYocWLF+vuu+92+J/5zjvv1MGDBy1M5l6GDh2qcePGKSIiQr6+vvbx+++/X9OnT7cwmdmmTZsm6fq/N3PmzFHBggXtx9LT07Vp0yZVq1bNqnhuxcPDQ02aNLE6hlt59913rY6AP3n++ecdPr969ap27NihNWvWaPDgwdaEcgN/nia7fv16DRkyRBMmTFCjRo0kSTExMRo+fLgmTJhgVUSno+g4CRsRWO/06dNZputIUkpKCtNFXGjXrl1auHBhlvESJUrozJkzFiRyD1OmTJF0/YrOrFmz5OnpaT/m5eWloKAgzZo1y6p4buWFF17QjBkzFBUVZXUUwBIDBw7MdnzGjBn6/vvvXZzGPT3//POaNWuWmjZtah9r1aqVfHx89NRTT2nPnj0WpnMeio6TsRDeOiEhIVq1apUGDBgg6Y9LtHPmzLG/mwHnK1SokE6ePJllY44dO3aoTJkyFqUy3+9vttx3331atmyZChcubHEi9zVo0CC1bdtWlSpVUnBwsPLmzetwnDUizrd06VItWbJEiYmJSktLczgWFxdnUSq0bt1aw4YN4wqcCxw8eFCFChXKMu7v768jR464PI+rsL20k1y+fFm9e/eWj4+P7rzzTiUmJkqSBgwYoIkTJ1qczj1MmDBBL7/8svr3769r165p6tSpatmypd59912NHz/e6nhuo0uXLhoyZIiSkpJks9mUkZGhLVu2aNCgQerZs6fV8Yy3YcMGSo7FnnvuOW3YsEF33HGHihYtKn9/f4cHnGvatGkKCwtTQECAduzYoQYNGqho0aI6dOiQWrdubXU8t7Z06VIVKVLE6hhu4a677lJERISSk5PtY8nJyRo8eLAaNGhgYTLnYntpJxk4cKC2bNmiqKgoPfjgg/rhhx9UsWJFffrppxo9erR27NhhdUS3cPDgQU2cOFHx8fG6dOmS6tWrpyFDhqhmzZpWR3MbaWlpevbZZzV//nylp6crT548Sk9PV7du3TR//nyHKVXIfenp6Zo/f76io6Oz3dr4q6++siiZ+/D19dWiRYvUtm1bq6O4pWrVqmnUqFHq2rWrfH19FR8fr4oVK2rkyJE6e/YsawVdoG7dug5TxjMzM5WUlKTTp0/rrbfe0lNPPWVhOveQkJCghx9+WPv371dgYKAk6dixY6pSpYpWrFihypUrW5zQOSg6TlK+fHn7Qvg//8OakJCgevXq6eLFi1ZHBFzq2LFj2rVrly5duqS6deuqSpUqVkdyC+Hh4Zo/f77atm2rUqVKZVmf9vtaHjhP+fLltXbtWjZ/sIiPj4/27Nmj8uXLq0SJEvryyy9Vu3ZtHThwQHfffbd++eUXqyMab8yYMQ6fe3h4qHjx4rr33nv5/8KFMjMz9eWXX2rv3r2SpOrVqys0NNTodcssGnESFsL/O2RkZCghISHbd7KbN29uUSr3FBgYqMDAQKWnp2vXrl06d+4cU6pcYNGiRVqyZInatGljdRS3NXr0aI0aNUrvvvuufHx8rI7jdkqWLKmzZ8+qfPnyKleunLZu3aratWvr8OHDN9x+Hbnn2rVrqlChglq1aqWAgACr47g1m82mli1bqmXLllZHcRmKjpOwEN56W7duVbdu3XT06NEsP8xsNpuxe8b/2zz//POqWbOmevfurfT0dN1zzz369ttv5ePjo88//1z33nuv1RGN5uXlZeyUhP+KadOm6eDBgwoICFBQUFCWzQhYDO9c999/v1auXKm6desqLCxML7zwgpYuXarvv/8+RzcWxe3JkyeP+vXrZ+yuXv91ycnJevvttzVy5EirozgFRcdJJkyYoNatW2v37t32hfC7d+/Wt99+q6+//trqeG6hX79+9sKZ3ZQduMbSpUvVo0cPSdJnn32mQ4cOae/evXr//ff1yiuvaMuWLRYnNNuLL76oqVOnavr06fw/YJEOHTrwd2+hd955x35F/9lnn1XRokX17bffqn379nr66actTuceGjRooB07dqh8+fJWR8FfJCUlacyYMcYWHdboOBEL4a1VoEABxcfH8262xfLly6eEhASVLVtWTz31lHx8fBQVFaXDhw+rdu3arFdzsocfflgbNmxQkSJFdOedd7K1MQCXW7JkiYYNG6YXXnhB9evXV4ECBRyO16pVy6Jk5vvhhx9uenzv3r3q2rWrsbNcuKLjRJUqVdLs2bOtjuG2GjZsqISEBIqOxQICArR7926VKlVKa9as0cyZMyVd34KdHdecr1ChQnr44YetjuHWKlasqO3bt6to0aIO4+fPn1e9evV06NAhi5K5j/Pnz2vbtm3Zrtdkm3vn69Kli6TrW63/zmazKTMzk6nkTlanTh373/Vf/fk1MBVFJxfl5J1pPz8/JyaBdP2eRS+++KKSkpJUs2bNLO9k8w6Sa4SFhemxxx6zTx8MDQ2VJH333XfstuMC3IjPekeOHMn2F7nU1FQdP37cgkTu5bPPPlP37t116dIl+fn5OfxSZ7PZKDou8PsNjOF6RYoU0aRJk9SiRYtsj//0009q166di1O5DkUnFxUqVOiWWzHvXjhfp06dJElPPvmkfYx3kFxv9OjRqlGjho4dO6ZHH31U3t7ekiRPT08NHTrU4nTu4dq1a9q4caMOHjyobt26ydfXVydOnJCfn58KFixodTxjrVy50v7x2rVrHW4Omp6erujoaFWoUMGKaG7lxRdf1JNPPqkJEyaw651FWJtjnfr16+vEiRM3fA3Onz9v9O6DrNHJRX/eZODIkSMaOnSonnjiCfsuazExMVqwYIEiIyPVq1cvq2K6jaNHj970OP/wwh0cPXpUDz74oBITE5Wamqr9+/erYsWKGjhwoFJTUzVr1iyrIxrLw8NDkrKdNpI3b14FBQXpjTfe0EMPPWRFPLdRoEAB7dq1SxUrVrQ6itv6c+n/M5vNpnz58qly5cqUfidZvny5UlJS7JsC/dW5c+e0cuVKY38vpeg4SYsWLdSnTx917drVYXzhwoV65513tHHjRmuCARaIjo5WdHR0tvPj582bZ1Eq99CxY0f5+vpq7ty5Klq0qP3mxRs3blTfvn114MABqyMar0KFCtq+fbuKFStmdRS39Mgjj6hLly567LHHrI7itjw8PLIt/H+eZdG0aVOtWLGC+6shVzF1zUliYmKyfac0JCREffr0sSCRe1i5cqVat26tvHnz3vAdpN+1b9/eRanc25gxYzR27FiFhISwzbcFvvnmG3377bfy8vJyGA8KCtLPP/9sUSr3kt36hPPnz6tQoUKuD+OG2rZtq8GDB2v37t3ZrtfkZ4Hzffnll3rllVc0fvx4NWjQQJK0bds2jRgxQsOHD5e/v7+efvppDRo0SHPnzrU4rZk2b96spk2bWh3D5bii4yRVq1ZVhw4dNGnSJIfxl156SZ9++qn27dtnUTKzeXh4KCkpSSVKlLBPG8kOa3Rcp1SpUpo0aZIef/xxq6O4pcKFC2vLli0KDg6Wr6+v/YrO5s2b1alTJyUnJ1sd0Xj/+9//FBQUpM6dO0uSHn30UX3yyScqVaqUVq9erdq1a1uc0Gz8LLBejRo19M4776hx48YO41u2bNFTTz2ln376SevXr9eTTz6pxMREi1KazcvLS2XKlFHXrl3Vo0cPBQcHWx3JJW78fz/+kSlTpujNN99UzZo11adPH/Xp00e1atXSm2++qSlTplgdz1gZGRkqUaKE/eMbPfjB5jppaWlZfrjBdVq2bKmoqCj75zabTZcuXdKoUaPUpk0b64K5kVmzZikwMFDS9Xe2169frzVr1qh169YaPHiwxenMx88C6x08eDDb3Wb9/Pzs26tXqVJFZ86ccXU0t3HixAm9+OKL+vrrr1WjRg3VqVNHr732mvE7P3JFx4mOHz+ut956S3v37pUkVa9eXf369bP/wAPcwZAhQ1SwYEGNGDHC6ihu6fjx42rVqpUyMzN14MABhYSE6MCBAypWrJg2bdpkf2MAzpM/f37t379fgYGBGjhwoK5cuaK3335b+/fvV8OGDXXu3DmrIwJO1bRpU/n6+uq9995T8eLFJUmnT59Wz549lZKSok2bNmn9+vV69tlnmfHiAocPH9bChQv10Ucfae/evWrevLm++uorq2M5BUUHRpk2bdotn/vnG5fBeQYOHKj33ntPtWrVUq1atbLMj588ebJFydzHtWvXtHjxYsXHx+vSpUuqV6+eunfvrvz581sdzS2ULl1aS5cuVePGjVW1alWNGzdOjz76qPbt26e77rorR/dgw+35+uuv9frrr2vPnj2SpODgYA0ePFjNmjWzOJl72Ldvnzp06KDDhw/b3+w9duyYKlasqE8//VR33HGHVqxYoV9//ZVpzi6Snp6uL774QiNGjNAPP/xg7NVNio4TnT9/XnPnzrX/w3rnnXfqySefdLiXAnLXrW5PabPZuBu5i9x33303PGaz2Yx9Fwn4XXh4uD7//HNVqVJFO3bs0JEjR1SwYEEtWrRIkyZNUlxcnNURjfbBBx8oLCxMjzzyiJo0aSLp+tqQ5cuXa/78+erWrZvFCd1DRkaG1q1bp/3790u6vpb5gQceuOkaKuS+LVu26MMPP9TSpUt15coVdejQQd27d9eDDz5odTSnoOg4yffff69WrVopf/789h1Gtm/frt9++03r1q1TvXr1LE4IwB1ERkYqICDA4ca50vVtvU+fPq0hQ4ZYlMx9XL16VVOnTtWxY8f0xBNPqG7dupKur+X09fVlJ04nq169up566im98MILDuOTJ0/W7Nmz7W9Gwno1a9bU6tWrmeLvBMOGDdOiRYt04sQJPfDAA+revbs6dOhg/E10KTpO0qxZM1WuXFmzZ89WnjzXd/G+du2a+vTpo0OHDmnTpk0WJ3QfaWlpOnz4sCpVqmR/LeB6CQkJOnjwoJo3b678+fPb750A5woKCtLChQuzbAjx3XffqUuXLtlufQyYxNvbWz/99JMqV67sMJ6QkKAaNWroypUrFiXDX/15Z0jkriZNmqh79+567LHH3OqeXvzW5yTff/+9Q8mRpDx58uill15SSEiIhcncx+XLlzVgwAAtWLBAkux3hB8wYIDKlCmjoUOHWpzQPfzyyy967LHHtGHDBtlsNh04cEAVK1ZU7969VbhwYb3xxhtWRzRaUlKSSpUqlWW8ePHiOnnypAWJ3MPf3cfrz7iPi3MFBgYqOjo6S9FZv349Vw7gNrZs2WJ1BEtQdJzEz89PiYmJqlatmsP4sWPH5Ovra1Eq9zJs2DDFx8dr48aNDnNPQ0NDNXr0aIqOi7zwwgvKmzevEhMTVb16dft4586dFRERQdFxssDAQG3ZsiXL+rUtW7aodOnSFqUyX8eOHW/pPO7j4nwvvviinnvuOe3cudN+ZXPLli2aP3++pk6danE6wHUOHDigDRs26NSpU8rIyHA4NnLkSItSORdFx0k6d+6s3r176/XXX3f4h3Xw4MHq2rWrxencw4oVK7R48WLdfffdDlOk7rzzTh08eNDCZO5l3bp1Wrt2rcqWLeswXqVKFR09etSiVO6jb9++ev7553X16lXdf//9kqTo6Gi99NJLevHFFy1OZ66//hIB6/Tv318lS5bUG2+8oSVLlki6vm5n8eLF6tChg8XpANeYPXu2+vfvr2LFiqlkyZIOvxfZbDaKDnLm9ddfl81mU8+ePXXt2jVJUt68edW/f39NnDjR4nTu4fTp09neIyQlJYW1IS6UkpKS7WLHs2fPytvb24JE7mXw4MH65Zdf9MwzzygtLU2SlC9fPg0ZMkTDhg2zOB3+jIXYzvPwww/r4YcftjoGYJlx48Zp/PjxbrcBDXv6OYmXl5emTp2qc+fOaefOndq5c6fOnj2rKVOm8Mudi4SEhGjVqlX2z38vN3PmzFGjRo2siuV2mjVrpvfee8/+uc1mU0ZGhiZNmnTTrafxz6Wnp+ubb77R0KFDdfr0aW3dulXx8fE6e/asse/e/ZcdOXJEV69etToGAAOdO3dOjz76qNUxXI4rOk5y4cIFpaenq0iRIqpZs6Z9/OzZs8qTJ4/8/PwsTOceJkyYoNatW2v37t26du2apk6dqt27d+vbb7/V119/bXU8tzFp0iS1aNFC33//vdLS0vTSSy/pp59+0tmzZ912caSreHp6qmXLltqzZ48qVKigu+66y+pIgEsUKVJE+/fvV7FixVS4cOGbXsU/e/asC5O5nz179mjr1q1q1KiRqlWrpr1792rq1KlKTU1Vjx497FNqJentt99WQECAhWnN9eijj2rdunXq16+f1VFciqLjJF26dFG7du30zDPPOIwvWbJEK1eu1OrVqy1K5j6aNm2qnTt3auLEiapZs6b9/kUxMTEO5RPOVaNGDe3fv1/Tp0+Xr6+vLl26pEceeUTPPvtstruBIXfVqFFDhw4duuWb6QIm+P0eRb9/zHRla6xZs0YdOnRQwYIFdfnyZS1fvlw9e/ZU7dq1lZGRoZYtW2rdunX2ssPNW52ncuXKGjFihLZu3aqaNWsqb968Dsefe+45i5I5F/fRcZIiRYpoy5YtDrtMSdLevXvVpEkT/fLLLxYlA+BO1qxZo2HDhunVV19V/fr1VaBAAYfjXF3+9+AeIjBN48aNdf/992vcuHFatGiRnnnmGfXv31/jx4+XdH131NjYWK1bt87ipOa72ZtdNptNhw4dcmEa16HoOEmBAgXsrfnPdu3apYYNG+ry5csWJXMfcXFxyps3r/01+PTTT/Xuu+8qODhYo0ePlpeXl8UJzfXDDz/c8rm1atVyYhJ4ePyxFPPP72r/fsNWtjb+96DoOIenp6dOnjyZZXOaX375RSVKlOD/ASfy9/dXbGysKleurIyMDHl7e2vbtm2qW7euJOnHH39UaGiokpKSLE4KUzF1zUkaNGigd955R2+++abD+KxZs1S/fn2LUrmXp59+WkOHDlXNmjV16NAhde7cWY888og+/vhjXb58WVFRUVZHNFadOnVks9nsv0z/7vf3Vf48xi8ZzrVhwwarIwCWutH7uampqbzh5QK//3vv4eGhfPnyyd/f337M19dXFy5csCqa28ruZ7GpKDpOMm7cOIWGhio+Pl4tWrSQdP3eFdu3b+cSrYvs379fderUkSR9/PHHuueee7Rw4UJt2bJFXbp0oeg40eHDh+0f79ixQ4MGDdLgwYPtu93FxMTojTfe0KRJk6yK6DbuueceqyPgFrEQO3dNmzZN0vVf5ubMmaOCBQvaj6Wnp2vTpk1ZbuqN3BUUFKQDBw6oUqVKkq7/21+uXDn78cTERNZqutB7772n1157TQcOHJAk3XHHHRo8eLAef/xxi5M5D0XHSZo0aaKYmBi99tprWrJkifLnz69atWpp7ty5qlKlitXx3EJmZqb9pn3r16/XQw89JOn6neLPnDljZTTjlS9f3v7xo48+qmnTpqlNmzb2sVq1aikwMFAjRoy45TvI4/Z98803evvtt3Xo0CF9/PHHKlOmjN5//31VqFBBTZs2tTqe0X777TfFxsaqSJEiCg4Odjh25coVLVmyRD179pTEQuzcNmXKFEnXfxbMmjVLnp6e9mNeXl4KCgrSrFmzrIrnFvr37+9w1b5GjRoOx7/44guHXdfgPJMnT9aIESMUHh6uJk2aSJI2b96sfv366cyZM3rhhRcsTugcrNGBse6//34FBgYqNDRUvXv31u7du1W5cmV9/fXX6tWrl44cOWJ1RLeQP39+xcXFZdmYY8+ePapXr55+++03i5K5h08++USPP/64unfvrvfff1+7d+9WxYoVNX36dK1evZodIJ1o//79atmypRITE2Wz2dS0aVMtWrTI/g52cnKySpcuzfRNJ7vvvvu0bNkyFS5c2OoogGUqVKigMWPG2N9Y+d2CBQs0evRoh5kYJuGGoU6UkZGh/fv3a/Pmzdq0aZPDA84XFRWluLg4hYeH65VXXlHlypUlSUuXLlXjxo0tTuc+qlevrsjISKWlpdnH0tLSFBkZmaX8IPeNGzdOs2bN0uzZsx22E23SpIni4uIsTGa+IUOGqEaNGjp16pT27dsnX19fNWnSRImJiVZHcysbNmyg5MDtnTx5MtvffRo3bqyTJ09akMg1uKLjJFu3blW3bt109OjRLAsh2enIWleuXJGnp2eWPeThHNu2bVO7du2UmZlp32Hthx9+kM1m02effaYGDRpYnNBsPj4+2r17t4KCghx29Tp06JCCg4N15coVqyMaKyAgQOvXr7fv/JiZmalnnnlGq1ev1oYNG1SgQAGu6LhAp06d1KBBAw0ZMsRhfNKkSdq+fbs+/vhji5IBrlOjRg1169ZNL7/8ssP4uHHjtHjxYu3atcuiZM7FGh0n6devn0JCQrRq1SqVKlXKLXa2+K/Ily+f1RHcSoMGDXTo0CF9+OGH2rt3rySpc+fO6tatW5Z7uiD3lSxZUgkJCQoKCnIY37x5M9sYO9lvv/2mPHn++DFrs9k0c+ZMhYeH2zdHgfNt2rRJo0ePzjLeunVrvfHGG64PBFhgzJgx6ty5szZt2mRfo7NlyxZFR0dryZIlFqdzHoqOkxw4cEBLly61T5eC66Wnp2vKlClasmSJEhMTHaZOSdLZs2ctSuZ+ChQooKeeeuqm57Rt21Zz5sxhB55c1rdvXw0cOFDz5s2TzWbTiRMnFBMTo0GDBmnEiBFWxzNatWrV9P3332eZojl9+nRJUvv27a2I5XYuXbqU7TbSefPm1cWLFy1IBLhep06d9N1332nKlClasWKFpOtTy/98XyMTsUbHSRo2bKiEhASrY7i1MWPGaPLkyercubMuXLigiIgIPfLII/Lw8Mj23T1Ya9OmTWxM4ARDhw5Vt27d1KJFC126dEnNmzdXnz599PTTT2vAgAFWxzPaww8/rI8++ijbY9OnT1fXrl1veI8X5J6aNWtq8eLFWcYXLVqUZSc8wGT169fXBx98oNjYWMXGxuqDDz4wuuRIrNFxmuXLl2v48OEaPHiwatasmWU9CHeDd75KlSpp2rRpatu2rXx9fbVz50772NatW5k28i/DXeGdKy0tTQkJCbp06ZKCg4Md7ikCmOyzzz7TI488om7dutm3Mo6OjtZHH32kjz/+mC3u4RZWr14tT09PtWrVymF87dq1ysjIUOvWrS1K5lwUHSfx8Mh6sezPd4pn8anzFShQQHv27FG5cuVUqlQprVq1SvXq1dOhQ4dUt25d7sb8L0PRcb5jx45Jun4vKcCdrFq1ShMmTNDOnTvt97UbNWoUN9SF26hVq5YmTpzocE87SVqzZo2GDBmi+Ph4i5I5F2t0nMTU/cj/S8qWLauTJ0+qXLlyqlSpktatW6d69epp+/bt8vb2tjoe4BLXrl3TmDFjNG3aNF26dEmSVLBgQQ0YMECjRo1i90G4hbZt26pt27ZWxwAsc+DAgWynalarVs3opRYUHSf5853hYY2HH35Y0dHRatiwoQYMGKAePXpo7ty5SkxMNPYOwMBfDRgwQMuWLdOkSZPUqFEjSVJMTIxGjx6tX375RTNnzrQ4IeB858+f19KlS3Xo0CENGjRIRYoUUVxcnAICAlSmTBmr4wFO5+/vr0OHDmXZgTMhIcHoHVCZuuZE77//vmbNmqXDhw8rJiZG5cuXV1RUlCpUqKAOHTpYHc/txMTEKCYmRlWqVFG7du2sjoO/YOqac/j7+2vRokVZ5l+vXr1aXbt2ZQonjPfDDz8oNDRU/v7+OnLkiPbt26eKFStq+PDhSkxM1HvvvWd1RMDpnn76acXExGj58uWqVKmSpOslp1OnTrrrrrs0Z84cixM6B7uuOcnMmTMVERGhNm3a6Pz58/Y1OYUKFVJUVJS14dxUo0aNFBERQcn5l3r55ZdVpEgRq2MYx9vbO8s7eJJUoUKFbLfcBUwTERGhJ554QgcOHHC4j1qbNm20adMmC5MBrjNp0iQVKFBA1apVU4UKFVShQgVVr15dRYsW1euvv251PKfhio6TBAcHa8KECerYsaPDO9U//vij7r33Xp05c8bqiG5h3759evPNN7Vnzx5J1/eMHzBggKpWrWpxMvfC1U3rjB07Vnv37tW7775rX5uWmpqq3r17q0qVKho1apTFCQHn8vf3V1xcnCpVquTw8/jo0aOqWrWqrly5YnVEwCUyMzP15ZdfKj4+3r4pR/Pmza2O5VSs0XGSw4cPZ7s3ube3t1JSUixI5H4++eQTdenSRSEhIfa1CVu3blWNGjW0aNEiderUyeKE7mHmzJkaOXKknn/+eY0fPz7L1U2KjnPt2LFD0dHRKlu2rGrXri1Jio+PV1pamlq0aKFHHnnEfu6yZcusigk4jbe3d7Y3Bt2/f7+KFy9uQSLAGjabTS1btlTLli1veE7NmjW1evVqY3bnpOg4SYUKFbRz584smxKsWbMmy12y4RwvvfSShg0bprFjxzqMjxo1Si+99BJFx0XefPNNzZ49Wx07dtTEiRPt4yEhIRo0aJCFydxDoUKFsvy3bsoPMOBWtG/fXmPHjtWSJUskXf9lLzExUUOGDOHnAPAXR44c0dWrV62OkWsoOk4SERGhZ599VleuXFFmZqa2bdumjz76SJGRkcYu+Pq3OXnypHr27JllvEePHnrttdcsSOSeuLpprXffffeWztuyZYtSU1PZeh3GeeONN/R///d/KlGihH777Tfdc889SkpKUqNGjTR+/Hir4wFwIoqOk/Tp00f58+fX8OHDdfnyZXXr1k2lS5fW1KlT1aVLF6vjuYV7771X33zzjSpXruwwvnnzZjVr1syiVO6Hq5v/Da1bt9bOnTvZ9Q7G8ff315dffqktW7YoPj5ely5dUr169RQaGmp1NABORtFxou7du6t79+66fPmyLl26pBIlSmQ5Z8uWLQoJCeFd1FyycuVK+8ft27fXkCFDFBsbq7vvvlvS9TU6H3/8scaMGWNVRLfD1c3/BvalgYmuXr2q/Pnza+fOnWrSpImaNGlidSQALsSuaxbz8/PjXdRc5OFxazum22w2+6J4ON+HH36o0aNH6+DBg5Kk0qVLa8yYMerdu7fFyfA77mMEU1WsWFHLly+3b8YB4MZM+1nAfXQsRs/MXRkZGbf0oOS4Vvfu3XXgwAFdunRJSUlJOn78OCUHgEu88sorevnll3X27FmrowBwMaauwe2ZtpXiv5mPj498fHysjgHAjUyfPl0JCQkqXbq0ypcvrwIFCjgcj4uLsygZ4HopKSlasmSJEhISVKpUKXXt2lVFixa1H3/77bcVEBBgYcLcRdGB2zNtK8V/g7p168pms93SufyS8e9wq68X8F/TsWNHqyMAlgkODtbmzZtVpEgRHTt2TM2bN9e5c+d0xx136ODBg3r11Ve1detWVahQQZLUrVs3ixPnLooOgFzHLxb/PUyjhalGjRpldQTAMnv37tW1a9ckScOGDVPp0qW1c+dO+fv769KlS3r44Yf1yiuvaOHChRYndQ6KjsV4FxUm4heLf5cLFy4oKSlJklSyZEn5+/tnOefXX391dSwAgAvFxMRo1qxZ9p8BBQsW1JgxY4y+7QlFx2K8iwp38f3332vPnj2Srl9Kr1+/vsWJzDdnzhxNnjxZ+/btcxivWrWqXnzxRTaEgLGKFCmi/fv3q1ixYipcuPBN31RkkwKY7vf//q9cuaJSpUo5HCtTpoxOnz5tRSyXoOg40bVr17Rx40YdPHhQ3bp1k6+vr06cOCE/Pz8VLFhQEu+iwnzHjx9X165dtWXLFhUqVEiSdP78eTVu3FiLFi1S2bJlrQ1oqNdee02jR4/Wc889p1atWtkXlyYnJ2vdunUaOHCgzp07p0GDBlmcFMh9U6ZMka+vryQpKirK2jCAxVq0aKE8efLo4sWL2rdvn2rUqGE/dvToUYfNCEzDfXSc5OjRo3rwwQeVmJio1NRU7d+/XxUrVtTAgQOVmpqqWbNmWR0R/59pe8b/2zz44IM6f/68FixYoKpVq0qS9u3bp7CwMPn5+WnNmjUWJzRT+fLl9dprr+mxxx7L9vjixYs1ePBgJSYmujgZAMBV/nqD9LvvvlutWrWyfz548GAdP35cH330kaujuQRFx0k6duwoX19fzZ07V0WLFrX/Ir1x40b17dtXBw4csDqi0c6cOaN58+YpJibGYW1C48aN9cQTT6h48eL2cxcuXKgOHTpk2XIUuSN//vz69ttvVbduXYfx2NhYNWvWTJcvX7Yomdny58+vuLg4Va9ePdvju3fvVkhICH//MNLFixdv+Vw/Pz8nJgFgJaauOck333yjb7/9Vl5eXg7jQUFB+vnnny1K5R62b9+uVq1aycfHR6GhobrjjjskXZ+yM23aNE2cOFFr165VSEiIJPO2Uvy3CQwMzHb77vT0dJUuXdqCRO7hrrvu0sSJEzV37lzlyeP4T316err+97//6a677rIoHeBchQoVuuXNfriBNGAuio6TZGRkZPuP5/Hjx+3zhuEcAwYM0KOPPqpZs2Zl+UGXmZmpfv36acCAAYqJibEooXt57bXXNGDAAM2YMcNeLr///nsNHDhQr7/+usXpzDV9+nS1atVKJUuWVPPmzR3W6GzatEleXl5at26dxSkB59iwYYP94yNHjmjo0KF64okn1KhRI0nXd59asGCBIiMjrYoIwAWYuuYknTt3lr+/v9555x35+vrqhx9+UPHixdWhQweVK1dO7777rtURjZU/f37t2LFD1apVy/b43r17VbduXf32228uTuaeChcurMuXL+vatWv2Kwu/f/zX6YLsfpS7fv31V33wwQfaunWrwxTORo0aqVu3bkzZgVto0aKF+vTpo65duzqML1y4UO+88442btxoTTAATkfRcZLjx4+rVatWyszM1IEDBxQSEqIDBw6oWLFi2rRpk0qUKGF1RGNVqFBBY8aMUc+ePbM9/t5772nkyJE6cuSIa4O5qQULFtzyub169XJiEgDuyMfHR/Hx8apSpYrD+P79+1WnTh3WqQEGY+qak5QtW1bx8fFavHix4uPjdenSJfXu3Vvdu3dX/vz5rY5ntEGDBumpp55SbGysWrRo4TBlJzo6WrNnz2bKlAtRXv6drl69qpMnT6pcuXJWRwGcKjAwULNnz9akSZMcxufMmaPAwECLUgFwBa7owEiLFy/WlClTFBsba18r5enpqfr16ysiIuKGW+7CeU6dOqVTp04pIyPDYbxWrVoWJXJv8fHxqlevHguxYbzVq1erU6dOqly5sho2bChJ2rZtmw4cOKBPPvlEbdq0sTghAGeh6DjJggULVKxYMbVt21aS9NJLL+mdd95RcHCwPvroI5UvX97ihO7h6tWrOnPmjCSpWLFiyps3r8WJ3E9sbKx69eqlPXv26K//3NhsNn7RtghFB+7k2LFjmjlzpvbu3StJql69uvr168cVHcBwFB0nqVq1qmbOnKn7779fMTExatGihaKiovT5558rT548WrZsmdURAZeoXbu2KlWqpCFDhiggICDLTniUfueoV6/eTY//9ttv2r9/P0UHAGAsio6T+Pj4aO/evSpXrpyGDBmikydP6r333tNPP/2ke++9V6dPn7Y6IuASvr6+2rFjhypXrmx1FLeSL18+denSRRUqVMj2+MmTJzV79myKDoz0ww8/qEaNGvLw8NAPP/xw03OZPguYi80InKRgwYL65ZdfVK5cOa1bt04RERGSrv/ywbbGcCctWrRQfHw8RcfFatSooYYNG6p///7ZHt+5c6dmz57t4lSAa9SpU0dJSUkqUaKE6tSpI5vNlmXqrMT0WcB0FB0neeCBB9SnTx/VrVtX+/fvty92/OmnnxQUFGRtOMCF5syZo169eunHH39UjRo1sqyTat++vUXJzNakSRPt27fvhsd9fX3VvHlzFyYCXOfw4cMqXry4/WMA7ompa05y/vx5DR8+XMeOHVP//v314IMPSpJGjRolLy8vvfLKKxYnBFzjs88+0+OPP66LFy9mOca7qQAAwFkoOgCcKigoSA899JBGjBhhv6cR/n2eeeYZjR07VsWKFbM6CpDr9u3bpzfffFN79uyRdH3XtQEDBqhq1aoWJwPgTBQdJ9m0adNNjzNlBO7C19dXO3fuVKVKlayOgpvw8/PTzp07VbFiRaujALnqk08+UZcuXRQSEqJGjRpJkrZu3art27dr0aJF6tSpk8UJATgLRcdJPDw8soz9eVtdpuvAXfTq1UvNmjVTnz59rI6Cm/D19VV8fDxFB8apVKmSunfvrrFjxzqMjxo1Sh988IEOHjxoUTIAzsZmBE5y7tw5h8+vXr2qHTt2aMSIERo/frxFqQDXu+OOOzRs2DBt3rxZNWvWzLIZwXPPPWdRMgDu4OTJk+rZs2eW8R49eui1116zIBEAV6HoOIm/v3+WsQceeEBeXl6KiIhQbGysBakA15szZ44KFiyor7/+Wl9//bXDMZvNRtEB4FT33nuvvvnmmyxb3G/evFnNmjWzKBUAV6DouFhAQMBNt3wFTMPWrgBcbeXKlfaP27dvryFDhig2NlZ33323pOtrdD7++GONGTPGqogAXIA1Ok7y1zsxZ2Zm6uTJk5o4caKuXbumzZs3W5QMALJijQ5Mkt062eywxT1gNq7oOMmN7sR89913a968eRalAlwjIiJCr776qgoUKKCIiIibnjt58mQXpXI/165d04QJE/Tkk0+qbNmyNz23R48e8vPzc1EywLkyMjKsjgDgX4ArOk5y9OhRh889PDxUvHhx5cuXz6JEgOvcd999Wr58uQoVKqT77rvvhufZbDZ99dVXLkzmfnx9fbVr1y4FBQVZHQX4V6tZs6ZWr16twMBAq6MAyCUUHYvxDysAZ+rQoYMeeeQR9erVy+oowL8a0zcB8zB1zWJHjhzR1atXrY4BuMzFixf11VdfqVq1aqpWrZrVcYzXunVrDR06VLt27VL9+vVVoEABh+Pt27e3KBkAAM7FFR2L8Q4STPfYY4+pefPmCg8P12+//abatWvryJEjyszM5K7kLnCzRdksxAb+wM9jwDy3ti0JANymTZs22e9VsXz5cmVmZur8+fOaNm2axo0bZ3E682VkZNzwQckBAJiMogPAqS5cuKAiRYpIktasWaNOnTrJx8dHbdu21YEDByxO516uXLlidQQAAFyGogPAqQIDAxUTE6OUlBStWbNGLVu2lCSdO3eOXQhdID09Xa+++qrKlCmjggUL6tChQ5KkESNGaO7cuRanA1yPGfuA+6DoAHCq559/Xt27d1fZsmVVunRp3XvvvZKuT2mrWbOmteHcwPjx4zV//nxNmjRJXl5e9vEaNWpozpw5FiYDrOHt7a09e/ZkGX/77bcVEBBgQSIAzsJmBBZbuHChOnTokGUnJMAksbGxSkxM1AMPPKCCBQtKklatWqVChQqpSZMmFqczW+XKlfX222+rRYsWDout9+7dq0aNGuncuXNWRwSc4kY3K546dap69OihokWLSuKmxYDJKDpOFB0drSlTptjfOapevbqef/55hYaGWpwM+Pfx8/PTzp072fEol+XPn1979+5V+fLlHYrO7t271aBBA126dMnqiIBTeHh4qHbt2ipUqJDD+Ndff62QkBAVKFCAmxYDhmPqmpO89dZbevDBB+Xr66uBAwdq4MCB8vPzU5s2bTRjxgyr4wH/Orzn4hzBwcH65ptvsowvXbpUdevWtSAR4BoTJkzQhQsXNGLECG3YsMH+8PT01Pz587VhwwZKDmA4bhjqJBMmTNCUKVMUHh5uH3vuuefUpEkTTZgwQc8++6yF6QC4i5EjR6pXr176+eeflZGRoWXLlmnfvn1677339Pnnn1sdD3CaoUOHqkWLFurRo4fatWunyMhI5c2b1+pYAFyIKzpOcv78eT344INZxlu2bKkLFy5YkAiAO+rQoYM+++wzrV+/XgUKFNDIkSO1Z88effbZZ3rggQesjgc41V133aXY2FidPn1aISEh+vHHH2Wz2ayOBcBFuKLjJO3bt9fy5cs1ePBgh/FPP/1UDz30kEWpALijZs2a6csvv7Q6BmCJggULasGCBVq0aJFCQ0O5US7gRig6uWjatGn2j4ODgzV+/Hht3LhRjRo1kiRt3bpVW7Zs0YsvvmhVROBfi3dZAThTly5d1LRpU8XGxqp8+fJWxwHgAuy6losqVKhwS+fZbDb7TfsAXPfnHcHwzxQuXPiWi+PZs2ednAYAAGtwRScXHT582OoIwH/GsWPHNGrUKM2bN0+S9MUXX6hMmTIWpzJDVFSU/eNffvlF48aNU6tWrexXl2NiYrR27VqNGDHCooQAADgfV3QAWCI+Pl716tVjvryTderUSffdd5/DDpCSNH36dK1fv14rVqywJhgAAE5G0XGSJ5988qbHf38XGzDVypUrb3r80KFDevHFFyk6TlawYEHt3LlTlStXdhhPSEhQnTp1uGEoAMBYTF1zknPnzjl8fvXqVf344486f/687r//fotSAa7TsWNH2Wy2m94IlA0InK9o0aL69NNPs2yC8umnn6po0aIWpQIAwPkoOk6yfPnyLGMZGRnq37+/KlWqZEEiwLVKlSqlt956Sx06dMj2+M6dO1W/fn0Xp3I/Y8aMUZ8+fbRx40Y1bNhQkvTdd99pzZo1mj17tsXpAABwHm4Y6kIeHh6KiIjQlClTrI4COF39+vUVGxt7w+N/d7UHueOJJ57Qli1b5Ofnp2XLlmnZsmXy8/PT5s2b9cQTT1gdDwAAp+GKjosdPHhQ165dszoG4HSDBw9WSkrKDY9XrlxZGzZscGEi99WwYUN9+OGHVscAAMCl2IzASSIiIhw+z8zM1MmTJ7Vq1Sr16tVL06dPtygZAHeTnp6uFStWaM+ePZKkO++8U+3bt5enp6fFyQAAcB6KjpPcd999Dp97eHioePHiuv/++/Xkk08qTx4upgFwvoSEBLVt21bHjx9X1apVJUn79u1TYGCgVq1axZpBAICxKDoAYLA2bdooMzNTH374oYoUKSLp+k1Ee/ToIQ8PD61atcrihAAAOAdFBwAMVqBAAW3dulU1a9Z0GI+Pj1eTJk24jw4AwFjsuuYkycnJevzxx1W6dGnlyZNHnp6eDg8AcAVvb2/9+uuvWcYvXbokLy8vCxIBAOAaLBRxkieeeEKJiYkaMWKESpUqxY0RAVjioYce0lNPPaW5c+eqQYMGkq7fR6dfv35q3769xekAAHAepq45ia+vr7755hvVqVPH6igA3Nj58+fVq1cvffbZZ8qbN68k6dq1a2rfvr3mz58vf39/ixMCAOAcXNFxksDAQG6GCMByhQoV0qeffqqEhAT79tLVq1dX5cqVLU4GAIBzcUXHSdatW6c33nhDb7/9toKCgqyOAwAAALgVik4uKly4sMNanJSUFF27dk0+Pj72KSO/O3v2rKvjAXBDnTp1UoMGDTRkyBCH8UmTJmn79u36+OOPLUoGAIBzUXRy0YIFC2753F69ejkxCQBcV7x4cX311VdZtpfetWuXQkNDlZycbFEyAACcizU6ueh2ysvEiRPVr18/FSpUKPcDAXB7N9pGOm/evLp48aIFiQAAcA3uo2OxCRMmMI0NgNPUrFlTixcvzjK+aNEiBQcHW5AIAADX4IqOxZg5CMCZRowYoUceeUQHDx7U/fffL0mKjo7WRx99xPocAIDRKDoAYLB27dppxYoVmjBhgpYuXar8+fOrVq1aWr9+ve655x6r4wEA4DRsRmAxX19fxcfHq2LFilZHAQAAAIzBFR0AcANpaWk6deqUMjIyHMbLlStnUSIAAJyLogMABjtw4ICefPJJffvttw7jmZmZstlsSk9PtygZAADORdHJRREREXr11VdVoEABbdq0SY0bN1aePDf/K27WrJny58/vooQA3M0TTzyhPHny6PPPP1epUqUcbmoMAIDJWKOTi/Lmzavjx48rICBAnp6eOnnypEqUKGF1LABurECBAoqNjVW1atWsjgIAgEtxRScXBQUFadq0aWrZsqUyMzMVExOjwoULZ3tu8+bNXZwOgDsKDg7WmTNnrI4BAIDLcUUnF61YsUL9+vXTqVOnZLPZbniPHObFA3CVr776SsOHD9eECRNUs2ZN5c2b1+G4n5+fRckAAHAuio4TXLp0SX5+ftq3b98Np675+/u7OBUAd+Th4SFJWdbmsBkBAMB0TF1zgoIFC2rDhg2qUKHC325GAADOtGHDBqsjAABgCa7oOMmNNiP45ZdfVKJECd5FBQAAAJzIw+oAprpRf0xNTZWXl5eL0wBwZ99884169Oihxo0b6+eff5Ykvf/++9q8ebPFyQAAcB7mVeWyadOmSbo+H37OnDkqWLCg/Vh6ero2bdrENq8AXOaTTz7R448/ru7duysuLk6pqamSpAsXLmjChAlavXq1xQkBAHAOpq7lsgoVKkiSjh49qrJly8rT09N+zMvLS0FBQRo7dqwaNmxoVUQAbqRu3bp64YUX1LNnT/n6+io+Pl4VK1bUjh071Lp1ayUlJVkdEQAAp+CKTi47fPiwJOm+++7TsmXLbngfHQBwhX379mV73y5/f3+dP3/e9YEAAHAR1ug4yYYNG26p5Pj5+enQoUMuSATAHZUsWVIJCQlZxjdv3qyKFStakAgAANeg6FiMmYMAnKlv374aOHCgvvvuO9lsNp04cUIffvihBg0apP79+1sdDwAAp2HqGgAYbOjQocrIyFCLFi10+fJlNW/eXN7e3ho0aJAGDBhgdTwAAJyGzQgs9ufFwQDgLGlpaUpISNClS5cUHBzssCOkJB0/flylS5eWhwcX+gEAZuCKDgC4AS8vLwUHB9/weHBwsHbu3MmbLgAAY/DWncVsNpvVEQCA9YIAAONQdCzGLxcAAABA7qPoWOyLL75QmTJlrI4BAAAAGIWik8vi4uLsNw2VpPfff19NmjRRYGCgmjZtqkWLFjmc37RpU3l7e7s6JgAAAGA0ik4uCwsL08GDByVJc+bM0dNPP62QkBC98soruuuuu9S3b1/NmzfP4pQA4Ij1ggAA07DrWi47cOCAqlSpIkl66623NHXqVPXt29d+/K677tL48eP15JNPWhURALJgvSAAwDRc0cllPj4+OnPmjCTp559/VoMGDRyON2zY0GFqGwC4QkJCgtauXavffvtNUtZis3v3bpUvX96KaAAAOAVFJ5e1bt1aM2fOlCTdc889Wrp0qcPxJUuWqHLlylZEA+CGfvnlF4WGhuqOO+5QmzZtdPLkSUlS79699eKLL9rPCwwMlKenp1UxAQDIdbZM5ivkqhMnTqhJkyYqV66cQkJCNHPmTNWvX1/Vq1fXvn37tHXrVi1fvlxt2rSxOioAN9CzZ0+dOnVKc+bMUfXq1RUfH6+KFStq7dq1ioiI0E8//WR1RAAAnII1OrmsdOnS2rFjhyZOnKjPPvtMmZmZ2rZtm44dO6YmTZpoy5YtCgkJsTomADexbt06rV27VmXLlnUYr1Klio4ePWpRKgAAnI+i4wSFChXSxIkTNXHiRKujAHBzKSkp8vHxyTJ+9uxZtrYHABiNNToAYLBmzZrpvffes39us9mUkZGhSZMm6b777rMwGQAAzsUaHQAw2I8//qgWLVqoXr16+uqrr9S+fXv99NNPOnv2rLZs2aJKlSpZHREAAKeg6ACA4S5cuKDp06crPj5ely5dUr169fTss8+qVKlSVkcDAMBpKDoAAAAAjMMaHQAw2Jo1a7R582b75zNmzFCdOnXUrVs3nTt3zsJkAAA4F0UHAAw2ePBgXbx4UZK0a9cuRUREqE2bNjp8+LAiIiIsTgcAgPOwvTQAGOzw4cMKDg6WJH3yySdq166dJkyYoLi4OG5cDAAwGld0AMBgXl5eunz5siRp/fr1atmypSSpSJEi9is9AACYiCs6AGCwpk2bKiIiQk2aNNG2bdu0ePFiSdL+/ftVtmxZi9MBAOA8XNEBAINNnz5defLk0dKlSzVz5kyVKVNGkvTFF1/owQcftDgdAADOw/bSAAAAAIzD1DUAMFhiYuJNj5crV85FSQAAcC2u6ACAwTw8PGSz2W54PD093YVpAABwHa7oAIDBduzY4fD51atXtWPHDk2ePFnjx4+3KBUAAM7HFR0AcEOrVq3Sa6+9po0bN1odBQAAp2DXNQBwQ1WrVtX27dutjgEAgNMwdQ0ADPbXm4JmZmbq5MmTGj16tKpUqWJRKgAAnI+iAwAGK1SoUJbNCDIzMxUYGKhFixZZlAoAAOdjjQ4AGOzrr792+NzDw0PFixdX5cqVlScP73UBAMxF0QEAqG3btpozZ45KlSpldRQAAHIFmxEAALRp0yb99ttvVscAACDXUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAC9/PLLKlKkiNUxAADINdxHBwAMt2/fPr355pvas2ePJKl69eoaMGCAqlatanEyAACchys6AGCwTz75RDVq1FBsbKxq166t2rVrKy4uTjVq1NAnn3xidTwAAJyGKzoAYLBKlSqpe/fuGjt2rMP4qFGj9MEHH+jgwYMWJQMAwLkoOgBgMB8fH/3www+qXLmyw/iBAwdUu3ZtXb582aJkAAA4F1PXAMBg9957r7755pss45s3b1azZs0sSAQAgGvksToAACB3rVy50v5x+/btNWTIEMXGxuruu++WJG3dulUff/yxxowZY1VEAACcjqlrAGAYD49bu1hvs9mUnp7u5DQAAFiDogMAAADAOKzRAQAAAGAc1ugAgMH+uq30X40cOdJFSQAAcC2mrgGAwerWrevw+dWrV3X48GHlyZNHlSpVUlxcnEXJAABwLq7oAIDBduzYkWXs4sWLeuKJJ/Twww9bkAgAANfgig4AuKFdu3apXbt2OnLkiNVRAABwCjYjAAA3dOHCBV24cMHqGAAAOA1T1wDAYNOmTXP4PDMzUydPntT777+v1q1bW5QKAADnY+oaABisQoUKDp97eHioePHiuv/++zVs2DD5+vpalAwAAOei6AAAAAAwDmt0AAAAABiHNToAYLCUlBRNnDhR0dHROnXqlDIyMhyOHzp0yKJkAAA4F0UHAAzWp08fff3113r88cdVqlQp2Ww2qyMBAOASrNEBAIMVKlRIq1atUpMmTayOAgCAS7FGBwAMVrhwYRUpUsTqGAAAuBxFBwAM9uqrr2rkyJG6fPmy1VEAAHAppq4BgGHq1q3rsBYnISFBmZmZCgoKUt68eR3OjYuLc3U8AABcgs0IAMAwHTt2tDoCAACW44oOAEAfffSR2rdvrwIFClgdBQCAXEHRAQDIz89PO3fuVMWKFa2OAgBArmAzAgCAeM8LAGAaig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6ACAYaZNm6YrV65IkhITE29po4Hy5ctnuZkoAAD/ZWwvDQCGyZMnj06cOKESJUrI09NTJ0+eVIkSJayOBQCAS+WxOgAAIHeVLl1an3zyidq0aaPMzEwdP37cfoXnr8qVK+fidAAAuAZXdADAMO+8844GDBiga9eu3fCczMxM2Ww2paenuzAZAACuQ9EBAAP9+uuvOnr0qGrVqqX169eraNGi2Z5Xu3ZtFycDAMA1KDoAYLAFCxaoS5cu8vb2tjoKAAAuxa5rAGCwMWPG6NKlS1nGz58/r4oVK1qQCAAA16DoAIDBjhw5ku06nNTUVP38888WJAIAwDXYdQ0ADLRy5Ur7x2vXrpW/v7/98/T0dEVHRysoKMiCZAAAuAZrdADAQB4e1y/Y22y2LDcMzZs3r4KCgvTGG2/ooYcesiIeAABOR9EBAINVqFBB27dvV7FixayOAgCAS7FGBwAMdvjw4VsqOTVr1tSxY8dckAgAANeg6AAAdOTIEV29etXqGAAA5BqKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcADDRgwAB98803t3z+22+/rYCAACcmAgDAtbhhKAAYyMPDQzabTZUqVVLv3r3Vq1cvlSxZ0upYAAC4DFd0AMBQ69atU5s2bfT666+rXLly6tChgz7//HNlZGRYHQ0AAKej6ACAoWrWrKmoqCidOHFCH3zwgVJTU9WxY0cFBgbqlVdeUUJCgtURAQBwGqauAYCBPDw8lJSUpBIlSjiMJyYmat68eZo/f76OHTum9PR0ixICAOBcFB0AMNCNis7vMjMztX79ej3wwAMuTgYAgGswdQ0ADFS+fHl5enre8LjNZqPkAACMxhUdAAAAAMbhig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYJz/B876xgBBkDB7AAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Saving and loading our trained models\n",
        "\n",
        "There are two main formats to save a model in TensorFlow:\n",
        "\n",
        "1. The `SavedModel` format (this is default when using tensorflow)\n",
        "2. The `HDF5` format"
      ],
      "metadata": {
        "id": "U6aP8jJqNSaF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Save TF Hub Sentence encoder model to HDF5 format\n",
        "model_6.save(\"model_6.h5\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hJ3Hscjh--p9",
        "outputId": "e2f7e8fc-5636-4c5f-f79a-5e81ff35ebed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
            "  saving_api.save_model(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load model with our custom Hub Layer (required HDF5 format)\n",
        "import tensorflow_hub as hub\n",
        "model_6_loaded = tf.keras.models.load_model(\"model_6.h5\",\n",
        "                                             custom_objects={\"KerasLayer\": hub.KerasLayer})"
      ],
      "metadata": {
        "id": "mXT9aQjA_UQ6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# How does our loaded model perform ?\n",
        "model_6_loaded.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j1_9TJ1LABrL",
        "outputId": "a312bafd-b768-4e70-88c7-5b6d5a0c4099"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 20ms/step - loss: 0.4296 - accuracy: 0.8058\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.42957767844200134, 0.8057742714881897]"
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_6_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h30epMx5AT9p",
        "outputId": "4899cc2d-dfe3-422f-bde0-e640d7cfaf72"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 80.5774278215223,\n",
              " 'precision': 0.8057420069627774,\n",
              " 'recall': 0.8057742782152231,\n",
              " 'f1': 0.8044743227851235}"
            ]
          },
          "metadata": {},
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let's save to the `SavedModel` format..."
      ],
      "metadata": {
        "id": "Tls3_GT3Ag6o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Save TF Hub Sentence Encoder model to SavedModel format\n",
        "model_6.save(\"model_6_saved_model\")"
      ],
      "metadata": {
        "id": "1iHuOyVcAsKE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load in a model from the SavedModel format\n",
        "model_6_saved_model = tf.keras.models.load_model(\"model_6_saved_model\")"
      ],
      "metadata": {
        "id": "C-rkHCdoBeVy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluate model in SavedModel format\n",
        "model_6_saved_model.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P62Glub0B3r4",
        "outputId": "f3272e54-8d09-4257-bf9f-6e44809d840c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 10ms/step - loss: 0.4296 - accuracy: 0.8058\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.42957767844200134, 0.8057742714881897]"
            ]
          },
          "metadata": {},
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Find the most wrong examples\n",
        "\n",
        "* If our best model still isn't perfect, what examples is it getting wrong ?\n",
        "* And of these wrong examples which ones is it getting *most* wrong (those with prediction probabilities closest to the opposite class)\n",
        "\n",
        "For example if a sample should have a label of 0 but our model predicts a prediction probability of 0.999 (really close to 1) and vice versa."
      ],
      "metadata": {
        "id": "8yTzOwCfCJYM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Download a pretrained model from Google Storage\n",
        "!wget https://storage.googleapis.com/ztm_tf_course/08_model_6_USE_feature_extractor.zip\n",
        "!unzip 08_model_6_USE_feature_extractor.zip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bgbBrMxyEM2l",
        "outputId": "3c8cb537-ac12-4b3f-9be6-57dac3469ed5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-08-26 16:06:31--  https://storage.googleapis.com/ztm_tf_course/08_model_6_USE_feature_extractor.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 142.251.2.207, 142.250.141.207, 74.125.137.207, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|142.251.2.207|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 960779165 (916M) [application/zip]\n",
            "Saving to: ‘08_model_6_USE_feature_extractor.zip’\n",
            "\n",
            "08_model_6_USE_feat 100%[===================>] 916.27M  50.7MB/s    in 21s     \n",
            "\n",
            "2024-08-26 16:06:52 (43.1 MB/s) - ‘08_model_6_USE_feature_extractor.zip’ saved [960779165/960779165]\n",
            "\n",
            "Archive:  08_model_6_USE_feature_extractor.zip\n",
            "   creating: 08_model_6_USE_feature_extractor/\n",
            "   creating: 08_model_6_USE_feature_extractor/assets/\n",
            "   creating: 08_model_6_USE_feature_extractor/variables/\n",
            "  inflating: 08_model_6_USE_feature_extractor/variables/variables.data-00000-of-00001  \n",
            "  inflating: 08_model_6_USE_feature_extractor/variables/variables.index  \n",
            "  inflating: 08_model_6_USE_feature_extractor/saved_model.pb  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import previously trained model from Google Storage\n",
        "model_6_pretrained = tf.keras.models.load_model(\"08_model_6_USE_feature_extractor\")\n",
        "model_6_pretrained.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TvM2qacNE4jS",
        "outputId": "92979640-dd89-45c2-800e-f3f6aff7e92e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:SavedModel saved prior to TF 2.5 detected when loading Keras model. Please ensure that you are saving the model with model.save() or tf.keras.models.save_model(), *NOT* tf.saved_model.save(). To confirm, there should be a file named \"keras_metadata.pb\" in the SavedModel directory.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 10ms/step - loss: 0.3872 - accuracy: 0.8241\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.38718459010124207, 0.8241469860076904]"
            ]
          },
          "metadata": {},
          "execution_count": 107
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions with the loaded model from Google Storage\n",
        "model_6_pretrained_pred_probs = model_6_pretrained.predict(val_sentences)\n",
        "model_6_pretrained_preds = tf.squeeze(tf.round(model_6_pretrained_pred_probs))\n",
        "model_6_pretrained_preds[:10] # these should be in label format"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FcnROzFuFKh2",
        "outputId": "fe0b589d-5eac-45ee-d5dd-43c01c4dae8f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 10ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 1., 0., 0., 0., 1., 1., 0., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a DataFrame with validation sentences, validation labels and best performing model prediction labels + probabilities\n",
        "val_df = pd.DataFrame({\"text\": val_sentences,\n",
        "                       \"target\": val_labels,\n",
        "                       \"pred\": model_6_pretrained_preds,\n",
        "                       \"pred_prob\": tf.squeeze(model_6_pretrained_pred_probs)})\n",
        "val_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "8xHuWDm6D3zW",
        "outputId": "6aa01408-93c3-4733-f65b-b01539f21fe0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  target  pred  pred_prob\n",
              "0  #IDFire Cherokee Road and Road 22 (Three Siste...       1   1.0   0.844321\n",
              "1  Beautiful lightning as seen from plane window ...       0   1.0   0.851977\n",
              "2  Why did God order obliteration of ancient Cana...       0   0.0   0.114461\n",
              "3  @Caitsroberts see U the night wee bArra to get...       0   0.0   0.063718\n",
              "4  Pandemonium In Aba As Woman Delivers Baby With...       1   0.0   0.247876"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3278e5c0-93fc-491c-92d2-3fc6a9f1bd3d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>#IDFire Cherokee Road and Road 22 (Three Siste...</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.844321</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Beautiful lightning as seen from plane window ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.851977</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Why did God order obliteration of ancient Cana...</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.114461</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@Caitsroberts see U the night wee bArra to get...</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.063718</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Pandemonium In Aba As Woman Delivers Baby With...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.247876</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3278e5c0-93fc-491c-92d2-3fc6a9f1bd3d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3278e5c0-93fc-491c-92d2-3fc6a9f1bd3d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3278e5c0-93fc-491c-92d2-3fc6a9f1bd3d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-b34bfb05-e515-4e26-891b-fb1779fb78a9\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b34bfb05-e515-4e26-891b-fb1779fb78a9')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-b34bfb05-e515-4e26-891b-fb1779fb78a9 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "val_df",
              "summary": "{\n  \"name\": \"val_df\",\n  \"rows\": 762,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 761,\n        \"samples\": [\n          \"Fuck Sleeping With Sirens.\",\n          \"BBC News - India rail crash: Trains derail in Madhya Pradesh flash flood http://t.co/WgmZmJ5imD\",\n          \"?? ice age: the meltdown\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0.0,\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred_prob\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 762,\n        \"samples\": [\n          0.035236891359090805,\n          0.25308623909950256\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the wrong predictions and sort by prediction probabilities\n",
        "most_wrong = val_df[val_df[\"target\"] != val_df[\"pred\"]].sort_values(\"pred_prob\", ascending=False)\n",
        "most_wrong[:10] # these are false positives"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "duQobzwcGTK-",
        "outputId": "3392df08-f790-4a0a-d5e6-8c9ffa0085cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text  target  pred  \\\n",
              "287  @TwopTwips make natural disaster reporting mor...       0   1.0   \n",
              "373  @CNN the End of Times are upon us. Famine War ...       0   1.0   \n",
              "697  FedEx will no longer transport bioterror patho...       0   1.0   \n",
              "548  Have you ever seen the President \\nwho killed ...       0   1.0   \n",
              "94   San Jose Mercury: Governor Brown allows parole...       0   1.0   \n",
              "239  Tales of the #trees #deep water loving #Lake T...       0   1.0   \n",
              "448  @noah_anyname That's where the concentration c...       0   1.0   \n",
              "1    Beautiful lightning as seen from plane window ...       0   1.0   \n",
              "100  Family mourns drowning of 'superhero' toddler ...       0   1.0   \n",
              "736  Governor allows parole for California school b...       0   1.0   \n",
              "\n",
              "     pred_prob  \n",
              "287   0.889624  \n",
              "373   0.881311  \n",
              "697   0.876982  \n",
              "548   0.874265  \n",
              "94    0.873231  \n",
              "239   0.869776  \n",
              "448   0.852300  \n",
              "1     0.851977  \n",
              "100   0.837567  \n",
              "736   0.817544  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-33f6adbf-b881-4aa1-9ec9-c959d437d1f7\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>287</th>\n",
              "      <td>@TwopTwips make natural disaster reporting mor...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.889624</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>373</th>\n",
              "      <td>@CNN the End of Times are upon us. Famine War ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.881311</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>697</th>\n",
              "      <td>FedEx will no longer transport bioterror patho...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.876982</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>548</th>\n",
              "      <td>Have you ever seen the President \\nwho killed ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.874265</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>San Jose Mercury: Governor Brown allows parole...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.873231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>239</th>\n",
              "      <td>Tales of the #trees #deep water loving #Lake T...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.869776</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>448</th>\n",
              "      <td>@noah_anyname That's where the concentration c...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.852300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Beautiful lightning as seen from plane window ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.851977</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>Family mourns drowning of 'superhero' toddler ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.837567</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>736</th>\n",
              "      <td>Governor allows parole for California school b...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.817544</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-33f6adbf-b881-4aa1-9ec9-c959d437d1f7')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-33f6adbf-b881-4aa1-9ec9-c959d437d1f7 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-33f6adbf-b881-4aa1-9ec9-c959d437d1f7');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d7420b8f-4467-4aa1-9c85-a9cfb98d0793\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d7420b8f-4467-4aa1-9c85-a9cfb98d0793')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d7420b8f-4467-4aa1-9c85-a9cfb98d0793 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"most_wrong[:10] # these are false positives\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"Family mourns drowning of 'superhero' toddler with rare epilepsy: Bradley Diebold suffered hundreds of epileptic\\u0089\\u00db_ http://t.co/unsayJDTu7\",\n          \"@CNN the End of Times are upon us. Famine War Death Plague. The presence is growing stronger.\",\n          \"Tales of the #trees #deep water loving #Lake Tahoe. And no #forest fires https://t.co/xuhMJ098Lq\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 0,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred_prob\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          0.8375670909881592\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 110
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "most_wrong.tail() # these are false negeatives"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "VbubimY3HX33",
        "outputId": "eace7211-13b2-4781-e57d-474726fd0a6d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text  target  pred  \\\n",
              "620                           Ain't no hoe in my blood       1   0.0   \n",
              "616  you can stab me in the back but I promise you'...       1   0.0   \n",
              "673  @leedsrouge Love what you picked! We're playin...       1   0.0   \n",
              "216  Reddit Will Now QuarantineÛ_ http://t.co/pkUA...       1   0.0   \n",
              "714  Oh the usual. Mass murder and world domination...       1   0.0   \n",
              "\n",
              "     pred_prob  \n",
              "620   0.076207  \n",
              "616   0.055746  \n",
              "673   0.053190  \n",
              "216   0.038949  \n",
              "714   0.036498  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bd821ff5-2452-4223-8d70-5ff9c17389b3\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>620</th>\n",
              "      <td>Ain't no hoe in my blood</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.076207</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>616</th>\n",
              "      <td>you can stab me in the back but I promise you'...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.055746</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>673</th>\n",
              "      <td>@leedsrouge Love what you picked! We're playin...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.053190</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>216</th>\n",
              "      <td>Reddit Will Now QuarantineÛ_ http://t.co/pkUA...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.038949</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>714</th>\n",
              "      <td>Oh the usual. Mass murder and world domination...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.036498</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bd821ff5-2452-4223-8d70-5ff9c17389b3')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bd821ff5-2452-4223-8d70-5ff9c17389b3 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bd821ff5-2452-4223-8d70-5ff9c17389b3');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4c546d94-82c3-4dfa-8e66-16347c7c56e3\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4c546d94-82c3-4dfa-8e66-16347c7c56e3')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4c546d94-82c3-4dfa-8e66-16347c7c56e3 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"most_wrong\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"you can stab me in the back but I promise you'll be the one bleeding\",\n          \"Oh the usual. Mass murder and world domination plans over coffee. How's your day going?\",\n          \"@leedsrouge Love what you picked! We're playing WORTH IT by FIFTH HARM/KID INK because of you! Listen &amp; Vote: http://t.co/0wrATkA2jL\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 1,\n        \"max\": 1,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred_prob\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          0.05574583262205124\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the false positives  (model predicted 1, when should've been 0)\n",
        "for row in most_wrong[:10].itertuples():\n",
        "  _, text, target, pred, pred_prob = row\n",
        "  print(f\"Target: {target}, Pred: {int(pred)}, Prob: {pred_prob}\")\n",
        "  print(f\"Text:\\n{text}\\n\")\n",
        "  print(\"----\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_7mWdl_pHuZm",
        "outputId": "cd2b5c64-1d87-4208-946d-23d608a5ae83"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target: 0, Pred: 1, Prob: 0.8896238803863525\n",
            "Text:\n",
            "@TwopTwips make natural disaster reporting more interesting by adding 'The' to headlines such as 'Rescuers are sifting through the wreckage'\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.8813107013702393\n",
            "Text:\n",
            "@CNN the End of Times are upon us. Famine War Death Plague. The presence is growing stronger.\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.8769821524620056\n",
            "Text:\n",
            "FedEx will no longer transport bioterror pathogens in wake of anthrax lab mishaps http://t.co/lHpgxc4b8J\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.8742649555206299\n",
            "Text:\n",
            "Have you ever seen the President \n",
            "who killed your wounded child?\n",
            "Or the man that crashed your sister's plane \n",
            "claimin' he was sent of God?\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.8732313513755798\n",
            "Text:\n",
            "San Jose Mercury: Governor Brown allows parole for California school bus hijacker  http://t.co/GpCeCp9kHv\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.8697755932807922\n",
            "Text:\n",
            "Tales of the #trees #deep water loving #Lake Tahoe. And no #forest fires https://t.co/xuhMJ098Lq\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.8523001074790955\n",
            "Text:\n",
            "@noah_anyname That's where the concentration camps and mass murder come in. \n",
            " \n",
            "EVERY. FUCKING. TIME.\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.8519769906997681\n",
            "Text:\n",
            "Beautiful lightning as seen from plane window http://t.co/5CwUyLnFUm http://t.co/1tyYqFz13D\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.8375670909881592\n",
            "Text:\n",
            "Family mourns drowning of 'superhero' toddler with rare epilepsy: Bradley Diebold suffered hundreds of epilepticÛ_ http://t.co/unsayJDTu7\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.8175444602966309\n",
            "Text:\n",
            "Governor allows parole for California school bus hijacker | Fresno Linked Local Network http://t.co/Sww0QsMxVM http://t.co/bcdP4gKokA\n",
            "\n",
            "----\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the false negeatives (model predicted 0, when should've been 1)\n",
        "for row in most_wrong[-10:].itertuples():\n",
        "  _, text, target, pred, pred_prob = row\n",
        "  print(f\"Target: {target}, Pred: {int(pred)}, Prob: {pred_prob}\")\n",
        "  print(f\"Text:\\n{text}\\n\")\n",
        "  print(\"----\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ozKMICeBIoLt",
        "outputId": "dca1c8e7-9d16-4615-afe8-152ff695fb5b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target: 1, Pred: 0, Prob: 0.09849575906991959\n",
            "Text:\n",
            "TodayÛªs storm will pass; let tomorrowÛªs light greet you with a kiss. Bask in this loving warmth; let your soul return to bliss.\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.09772361814975739\n",
            "Text:\n",
            "Jack Wilshere has poor injury recordand his off field behaviors doesn't help.#Arsenal\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.09160558879375458\n",
            "Text:\n",
            "Flattened thee striker\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.08508124947547913\n",
            "Text:\n",
            "I went to pick up my lunch today and the bartender was holding my change hostage because he wanted my number. ??\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.08042652159929276\n",
            "Text:\n",
            "VICTORINOX SWISS ARMY DATE WOMEN'S RUBBER MOP WATCH 241487 http://t.co/yFy3nkkcoH http://t.co/KNEhVvOHVK\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.0762074887752533\n",
            "Text:\n",
            "Ain't no hoe in my blood\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.05574583262205124\n",
            "Text:\n",
            "you can stab me in the back but I promise you'll be the one bleeding\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.05319014936685562\n",
            "Text:\n",
            "@leedsrouge Love what you picked! We're playing WORTH IT by FIFTH HARM/KID INK because of you! Listen &amp; Vote: http://t.co/0wrATkA2jL\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.03894945606589317\n",
            "Text:\n",
            "Reddit Will Now QuarantineÛ_ http://t.co/pkUAMXw6pm #onlinecommunities #reddit #amageddon #freespeech #Business http://t.co/PAWvNJ4sAP\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.03649798408150673\n",
            "Text:\n",
            "Oh the usual. Mass murder and world domination plans over coffee. How's your day going?\n",
            "\n",
            "----\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Making predictions on the test dataset"
      ],
      "metadata": {
        "id": "KIGiZmXsJROe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Making predictions on the test dataset and visualizing them\n",
        "test_sentences = test_df[\"text\"].to_list()\n",
        "test_samples = random.sample(test_sentences,10)\n",
        "for test_sample in test_samples:\n",
        "  pred_prob = tf.squeeze(model_6_pretrained.predict([test_sample])) # our model expects a list as input\n",
        "  pred = tf.round(pred_prob)\n",
        "  print(f\"Pred: {int(pred)}, Prob: {pred_prob}\")\n",
        "  print(f\"Text:\\n{test_sample}\\n\")\n",
        "  print(\"----\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iC2SruS5LA5E",
        "outputId": "6c931814-9df5-4d9e-b126-3f736ca8fe86"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 52ms/step\n",
            "Pred: 0, Prob: 0.24930278956890106\n",
            "Text:\n",
            "Remove the http://t.co/Ytao0lT144 and Linkury Browser Hijacker http://t.co/uxEKjeuIVp http://t.co/abvvRLFgDh\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 45ms/step\n",
            "Pred: 1, Prob: 0.8500266671180725\n",
            "Text:\n",
            "Go home California you're drunk. Natural selection has taken affect. Debt extreme wild fires water supply problems. You fail as a state.\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 34ms/step\n",
            "Pred: 1, Prob: 0.6729339957237244\n",
            "Text:\n",
            "Be safe and be prepare from emergency kits to evacuation. Alertness esp women and children who are also @gmanews  https://t.co/3GALBowItN\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 36ms/step\n",
            "Pred: 0, Prob: 0.36386638879776\n",
            "Text:\n",
            "omfg Johnny tsunami commercial\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 35ms/step\n",
            "Pred: 0, Prob: 0.07542961090803146\n",
            "Text:\n",
            "@AllOutAsh23_sN Heard you got obliterated and also that your brother/boyfriend ran out of the building crying last night. LOL. 'MD/VA!!!!'\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 35ms/step\n",
            "Pred: 0, Prob: 0.1905638873577118\n",
            "Text:\n",
            "Unlike the Donaldson dive into the stands no young boys were flattened in the Kyle Parker catch.\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 34ms/step\n",
            "Pred: 1, Prob: 0.9778728485107422\n",
            "Text:\n",
            "70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/JESoSSDjjH\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 51ms/step\n",
            "Pred: 1, Prob: 0.9250620603561401\n",
            "Text:\n",
            "Harper Woods policeman injured in fight after dead man found in garage: Ronnie Karjo 46 of Macomb Township http://t.co/tUoTICilBi\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 56ms/step\n",
            "Pred: 0, Prob: 0.04158369079232216\n",
            "Text:\n",
            "Reality continues to ruin my life.\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 60ms/step\n",
            "Pred: 0, Prob: 0.11554184556007385\n",
            "Text:\n",
            "? Stretcher in 5 min // Speaker Deck http://t.co/gXgJqQu3hU #pctool\n",
            "\n",
            "----\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## The speed/score tradeoff"
      ],
      "metadata": {
        "id": "ZCHgDs2cLNjl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's make a function to measure the time of prediction\n",
        "import time\n",
        "def pred_timer(model, samples):\n",
        "  \"\"\"\n",
        "  Times how long a model takes to make predictions on samples.\n",
        "  \"\"\"\n",
        "  start_time = time.perf_counter() # get start time\n",
        "  model.predict(samples) # make predictions\n",
        "  end_time = time.perf_counter() # get end time\n",
        "  total_time = end_time - start_time # calculate how long predictions took to make\n",
        "  time_per_pred = total_time/len(samples)\n",
        "  return total_time, time_per_pred"
      ],
      "metadata": {
        "id": "GOs2Z1-NN2Zm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate TF Hub Sentence Encoder time per pred\n",
        "model_6_total_pred_time, model_6_time_per_pred = pred_timer(model_6_pretrained, val_sentences)\n",
        "print(f\"Total time: {model_6_total_pred_time} seconds\")\n",
        "print(f\"Time per pred: {model_6_time_per_pred} seconds\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tZI8SD3RPcUi",
        "outputId": "d22c9e75-1dc6-4289-eba9-d71f98b4dcdf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 0s 16ms/step\n",
            "Total time: 0.5367101049999974 seconds\n",
            "Time per pred: 0.0007043439698162696 seconds\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate our baseline model time per pred\n",
        "baseline_total_pred_time, baseline_time_per_pred = pred_timer(model_0, val_sentences)\n",
        "print(f\"Total time: {baseline_total_pred_time} seconds\")\n",
        "print(f\"Time per pred: {baseline_time_per_pred} seconds\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VTG1Wyn_P34L",
        "outputId": "82563300-3020-480a-f217-d73c7ff59e30"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total time: 0.02959908599996197 seconds\n",
            "Time per pred: 3.884394488183986e-05 seconds\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_6_pretrained_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_6_pretrained_preds)\n",
        "model_6_pretrained_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nQ4Jz1vxSKSc",
        "outputId": "049851dd-9b34-43e2-b6f2-9a78e70e5512"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 82.41469816272966,\n",
              " 'precision': 0.8247806401490567,\n",
              " 'recall': 0.8241469816272966,\n",
              " 'f1': 0.8227682169021602}"
            ]
          },
          "metadata": {},
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Model's time per prediction vs F1 score for our baseline as well as our model 6 pretrained model\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.figure(figsize=(10,7))\n",
        "plt.scatter(baseline_time_per_pred, baseline_results[\"f1\"], label=\"baseline\")\n",
        "plt.scatter(model_6_time_per_pred, model_6_pretrained_results[\"f1\"], label=\"tf_hub_sentence_encoder\")\n",
        "plt.legend()\n",
        "plt.title(\"F1-score versus time per prediction\")\n",
        "plt.xlabel(\"Time per prediction (seconds)\")\n",
        "plt.ylabel(\"F1-score\");"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 641
        },
        "id": "2dOklMOrQDva",
        "outputId": "5197885c-f0af-4c5b-a37e-c38df0696bf9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA18AAAJwCAYAAAB2y9s+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABoi0lEQVR4nO3deVhV1eL/8c8BBGRORQZFcR7KIScix5JCLdK0nMoph2upOTRpznnVsjIb1O4tU+tqmWVZaWaOqTmiaCqSIuYQzgmKIQrr94c/z7cjoIi4AX2/nuc8edZeew17wb1+3PusYzPGGAEAAAAAbimn/B4AAAAAANwJCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAA3KDu3bsrNDQ0v4eBq6xatUo2m02rVq2yl+X1Ws2aNUs2m00HDhzIszYB3DkIXwDuWFf+EpXVa+jQofZ6S5cuVc+ePXXPPffI2dmZv3TfIf7880+NGTNGMTEx+T0U5IMJEybo22+/ze9hALjNuOT3AAAgv7322msqV66cQ9k999xj//PcuXM1b9481alTR8HBwVYPD/nkzz//1NixYxUaGqratWs7HPvoo4+UkZGRPwPDDcntWk2YMEFPPPGE2rRp41DepUsXdezYUW5ubnk0QgB3EsIXgDtey5YtVa9evWyPT5gwQR999JGKFCmiRx99VDt37rRwdHkjJSVFnp6e+T2M6yos4yxSpEh+DyFf3Kr1ycjIUFpamtzd3fO87bxeK2dnZzk7O+dpmwDuHDx2CADXERwcfFN/gTt79qwGDRqk0NBQubm5qWTJknrooYe0detWh3obN25Uq1atdNddd8nT01M1a9bUu+++61BnxYoVaty4sTw9PeXn56fWrVsrNjbWoc6YMWNks9m0e/dude7cWXfddZcaNWpkP/6///1PdevWVdGiRVWsWDF17NhRhw4duuYcvvrqK9lsNq1evTrTsf/85z+y2WwOoXTPnj164oknVKxYMbm7u6tevXr67rvvHM678tjn6tWr9dxzz6lkyZIqXbp0jq9ZaGiounfvnmk8zZo1U7NmzRzK3n//fd19993y8PDQXXfdpXr16mnu3LnZznfVqlWqX7++JKlHjx72x1FnzZolKfPniA4cOCCbzaa33npLU6dOVfny5eXh4aGHH35Yhw4dkjFG48aNU+nSpVW0aFG1bt1ap0+fztTvjz/+aF9fb29vPfLII9q1a1e247z6Wv7yyy/617/+peLFi8vHx0ddu3bVX3/9lat+unfvLi8vL8XHx6tVq1by9vbWU089le0Yrvzc7dmzR+3bt5ePj4+KFy+ugQMHKjU11aGuzWZT//79NWfOHN19991yc3PTkiVLJElHjhzRM888o4CAALm5uenuu+/WJ598kqm/w4cPq02bNvL09FTJkiU1ePBgXbhwIVO9rD7zlZGRoXfffVc1atSQu7u7/P391aJFC23ZssU+vpSUFM2ePdu+9ld+1rL7zNe0adPscwkODla/fv105swZhzrNmjXTPffco927d+uBBx6Qh4eHSpUqpUmTJmV7XQHcXrjzBeCOl5SUpJMnTzqUlShRIs/a79u3r7766iv1799f1atX16lTp7R27VrFxsaqTp06kqSff/5Zjz76qIKCgjRw4EAFBgYqNjZWP/zwgwYOHChJWrZsmVq2bKny5ctrzJgx+vvvv/X++++rYcOG2rp1a6a/YD755JOqVKmSJkyYIGOMJGn8+PEaOXKk2rdvr169eunEiRN6//331aRJE23btk1+fn5ZzuGRRx6Rl5eXvvzySzVt2tTh2Lx583T33XfbH9XctWuXGjZsqFKlSmno0KHy9PTUl19+qTZt2ujrr7/W448/7nD+c889J39/f40aNUopKSk5vmY59dFHH+n555/XE088YQ8CO3bs0MaNG9W5c+csz6lWrZpee+01jRo1Sn369FHjxo0lSffff/81+5ozZ47S0tI0YMAAnT59WpMmTVL79u314IMPatWqVXrllVe0b98+vf/++3rxxRcdQsVnn32mbt26KTIyUm+88YbOnz+v6dOnq1GjRtq2bVuOPmvYv39/+fn5acyYMYqLi9P06dP1xx9/2DeiuNF+Ll26pMjISDVq1EhvvfWWPDw8rjuG9u3bKzQ0VBMnTtSGDRv03nvv6a+//tKnn37qUG/FihX68ssv1b9/f5UoUUKhoaE6duyY7rvvPns48/f3148//qiePXsqOTlZgwYNkiT9/fffat68uQ4ePKjnn39ewcHB+uyzz7RixYrrjk+SevbsqVmzZqlly5bq1auXLl26pDVr1mjDhg2qV6+ePvvsM/Xq1UsNGjRQnz59JEkVKlTItr0xY8Zo7NixioiI0LPPPmu/9ps3b9a6desc/vHmr7/+UosWLdS2bVu1b99eX331lV555RXVqFFDLVu2zNH4ARRiBgDuUDNnzjSSsnxl55FHHjFly5a9oX58fX1Nv379sj1+6dIlU65cOVO2bFnz119/ORzLyMiw/7l27dqmZMmS5tSpU/ay7du3GycnJ9O1a1d72ejRo40k06lTJ4e2Dhw4YJydnc348eMdyn/77Tfj4uKSqfxqnTp1MiVLljSXLl2ylyUmJhonJyfz2muv2cuaN29uatSoYVJTUx3mcf/995tKlSrZy65c/0aNGjm0acz1r5kxxpQtW9Z069YtU3nTpk1N06ZN7e9bt25t7r777mu2lZXNmzcbSWbmzJmZjnXr1s3h5yAhIcFIMv7+/ubMmTP28mHDhhlJplatWubixYv28k6dOhlXV1f7NTp79qzx8/MzvXv3dujn6NGjxtfXN1P51a5cy7p165q0tDR7+aRJk4wks3Dhwhvup1u3bkaSGTp06DX7vuLKz91jjz3mUP7cc88ZSWb79u32MknGycnJ7Nq1y6Fuz549TVBQkDl58qRDeceOHY2vr685f/68McaYKVOmGEnmyy+/tNdJSUkxFStWNJLMypUrHebxz7VasWKFkWSef/75THP45++bp6dnlj9fV651QkKCMcaY48ePG1dXV/Pwww+b9PR0e70PPvjASDKffPKJvaxp06ZGkvn000/tZRcuXDCBgYGmXbt2mfoCcPvhsUMAd7ypU6fq559/dnjlJT8/P23cuFF//vlnlse3bdumhIQEDRo0KNOdpyt3KxITExUTE6Pu3burWLFi9uM1a9bUQw89pMWLF2dqt2/fvg7vFyxYoIyMDLVv314nT560vwIDA1WpUiWtXLnymvPo0KGDjh8/7rCN91dffaWMjAx16NBBknT69GmtWLFC7du319mzZ+19nDp1SpGRkdq7d6+OHDni0G7v3r0zfYbmetfsRvj5+enw4cPavHnzTbd1PU8++aR8fX3t78PCwiRJTz/9tFxcXBzK09LS7Nfi559/1pkzZ9SpUyeHtXF2dlZYWNh11+aKPn36ONxlefbZZ+Xi4mL/+chNP88+++wNXYN+/fo5vB8wYIAkZfoZbdq0qapXr25/b4zR119/raioKBljHMYXGRmppKQk+2OnixcvVlBQkJ544gn7+R4eHva7VNfy9ddfy2azafTo0ZmOXfl9uxHLli1TWlqaBg0aJCen//trVe/eveXj46NFixY51Pfy8tLTTz9tf+/q6qoGDRpo//79N9w3gMKHxw4B3PEaNGhwzQ03ciI9PV0nTpxwKCtWrJhcXV01adIkdevWTSEhIapbt65atWqlrl27qnz58pKk+Ph4SY47LF7tjz/+kCRVqVIl07Fq1arpp59+yrQZwtU7OO7du1fGGFWqVCnLPq73ubYWLVrI19dX8+bNU/PmzSVdfuSwdu3aqly5siRp3759MsZo5MiRGjlyZJbtHD9+XKVKlcp2nJKue81uxCuvvKJly5apQYMGqlixoh5++GF17txZDRs2vOG2rqdMmTIO768EsZCQkCzLr3wea+/evZKkBx98MMt2fXx8ctT/1Wvr5eWloKAg++eTbrQfFxcX++fwcurqMVSoUEFOTk6ZPiN19bqfOHFCZ86c0X//+1/997//zbLt48ePS7r8+1CxYsVMYSmr34+rxcfHKzg42OEfMW5Gdr+brq6uKl++vP34FaVLl8407rvuuks7duzIk/EAKNgIXwCQBw4dOpTpL5MrV65Us2bN1L59ezVu3FjffPONli5dqjfffFNvvPGGFixYcEs/41G0aFGH9xkZGbLZbPrxxx+z3K3Ny8vrmu25ubmpTZs2+uabbzRt2jQdO3ZM69at04QJExz6kKQXX3xRkZGRWbZTsWLFa45TUo6uWXZ3KdLT0x3mV61aNcXFxemHH37QkiVL9PXXX2vatGkaNWqUxo4de80536jsdsHLrtz8/8/iXblun332mQIDAzPV++dds5txo/24ubk53M3JjezWKaufT+nyXcJu3bpleU7NmjVvaiwFwfV+FgDc3ghfAJAHAgMDMz2uWKtWLfufg4KC9Nxzz+m5557T8ePHVadOHY0fP14tW7a0f5B/586dioiIyLL9smXLSpLi4uIyHduzZ49KlChx3S3AK1SoIGOMypUrZ79TdaM6dOig2bNna/ny5YqNjZUxxv7IoST7nakiRYpkO5ecutY1ky7fLbh6Nznp8p2Iq++QeXp6qkOHDurQoYPS0tLUtm1bjR8/XsOGDct2e/PcPIKWW1d+BkqWLHlT123v3r164IEH7O/PnTunxMREtWrVKk/7ud4Y/vkPEfv27VNGRsZ1Nwzx9/eXt7e30tPTrzu2smXLaufOnTLGOKxTVr8fV6tQoYJ++uknnT59+pp3v3K6/v/83fznz11aWpoSEhJu2XUGUDjxmS8AyAPu7u6KiIhweN11111KT09XUlKSQ92SJUsqODjYvi12nTp1VK5cOU2ZMiVTmLjyr+FBQUGqXbu2Zs+e7VBn586dWrp0qf0v19fStm1bOTs7a+zYsZn+ld0Yo1OnTl23jYiICBUrVkzz5s3TvHnz1KBBA4e/aJcsWVLNmjXTf/7zHyUmJmY6/+pHM7OSk2smXf5L9IYNG5SWlmYv++GHHzJtm3/1vFxdXVW9enUZY3Tx4sVsx3ElzGYV8PJaZGSkfHx8NGHChCzHlJPrJkn//e9/Hc6fPn26Ll26ZA+sedXPtUydOtXh/fvvvy9J173L6+zsrHbt2unrr7/O8rv0/jm2Vq1a6c8//9RXX31lLzt//ny2jyv+U7t27WSMyfKu5z9/Lzw9PXO09hEREXJ1ddV7773ncP6MGTOUlJSkRx555LptALhzcOcLAK5jx44d9u+o2rdvn5KSkvTvf/9b0uW7W1FRUdmee/bsWZUuXVpPPPGEatWqJS8vLy1btkybN2/W22+/LUlycnLS9OnTFRUVpdq1a6tHjx4KCgrSnj17tGvXLv3000+SpDfffFMtW7ZUeHi4evbsad9q3tfXV2PGjLnuPCpUqKB///vfGjZsmA4cOKA2bdrI29tbCQkJ+uabb9SnTx+9+OKL12yjSJEiatu2rb744gulpKTorbfeylRn6tSpatSokWrUqKHevXurfPnyOnbsmNavX6/Dhw9r+/bt1+wjJ9dMknr16qWvvvpKLVq0UPv27RUfH6///e9/mbYEf/jhhxUYGKiGDRsqICBAsbGx+uCDD/TII4/I29v7mtfLz89PH374oby9veXp6amwsLAsP6N2s3x8fDR9+nR16dJFderUUceOHeXv76+DBw9q0aJFatiwoT744IPrtpOWlqbmzZurffv2iouL07Rp09SoUSM99thjedrPtSQkJOixxx5TixYttH79ev3vf/9T586dHe4EZ+f111/XypUrFRYWpt69e6t69eo6ffq0tm7dqmXLltm/G61379764IMP1LVrV0VHRysoKEifffZZjrbCf+CBB9SlSxe999572rt3r1q0aKGMjAytWbNGDzzwgPr37y9Jqlu3rpYtW6bJkycrODhY5cqVs2+g8k/+/v4aNmyYxo4dqxYtWuixxx6zX/v69es7bK4BAGw1D+COdWXL6M2bN+eoXlavrLai/qcLFy6Yl156ydSqVct4e3sbT09PU6tWLTNt2rRMddeuXWseeughe72aNWua999/36HOsmXLTMOGDU3RokWNj4+PiYqKMrt373aoc2XL7xMnTmQ5pq+//to0atTIeHp6Gk9PT1O1alXTr18/ExcXd825XPHzzz8bScZms5lDhw5lWSc+Pt507drVBAYGmiJFiphSpUqZRx991Hz11Vf2Otld/xu5Zm+//bYpVaqUcXNzMw0bNjRbtmzJtNX8f/7zH9OkSRNTvHhx4+bmZipUqGBeeuklk5SUdN25Lly40FSvXt24uLg4bDuf3Vbzb775psP5K1euNJLM/PnzHcqzm/vKlStNZGSk8fX1Ne7u7qZChQqme/fuZsuWLdcc55X2Vq9ebfr06WPuuusu4+XlZZ566imHrya4kX66detmPD09r3uNrrjyc7d7927zxBNPGG9vb3PXXXeZ/v37m7///tuhrqRsv0rg2LFjpl+/fiYkJMQUKVLEBAYGmubNm5v//ve/DvX++OMP89hjjxkPDw9TokQJM3DgQLNkyZLrbjVvzOWvd3jzzTdN1apVjaurq/H39zctW7Y00dHR9jp79uwxTZo0MUWLFnX4Xb96q/krPvjgA1O1alVTpEgRExAQYJ599tlMXx3RtGnTLL/2IKsxArg92YzhE54AABRms2bNUo8ePbR58+ab3rkzt6580fCJEyfy9EvKAeB2wme+AAAAAMAChC8AAAAAsADhCwAAAAAswGe+AAAAAMAC3PkCAAAAAAsQvgAAAADAAnzJci5lZGTozz//lLe3t2w2W34PBwAAAEA+Mcbo7NmzCg4OlpNT9ve3CF+59OeffyokJCS/hwEAAACggDh06JBKly6d7XHCVy55e3tLunyBfXx88nk0AAAAAPJLcnKyQkJC7BkhO4SvXLryqKGPjw/hCwAAAMB1P47EhhsAAAAAYAHCFwAAAABYgPAFAAAAABbgM1+3kDFGly5dUnp6en4PBbgjFSlSRM7Ozvk9DAAAAEmEr1smLS1NiYmJOn/+fH4PBbhj2Ww2lS5dWl5eXvk9FAAAAMLXrZCRkaGEhAQ5OzsrODhYrq6ufBEzYDFjjE6cOKHDhw+rUqVK3AEDAAD5jvB1C6SlpSkjI0MhISHy8PDI7+EAdyx/f38dOHBAFy9eJHwBAIB8x4Ybt5CTE5cXyE/ccQYAAAUJ6QAAAAAALED4AgAAAAALEL7goFmzZho0aFC+9d+9e3e1adOmwIwHAAAAyCtsuIECbcGCBSpSpEh+DwMAAAC4aYSvAiw9w2hTwmkdP5uqkt7ualCumJyd7qwNBIoVK5bfQwAAAADyBI8dFlBLdiaq0Rsr1OmjDRr4RYw6fbRBjd5YoSU7E29535cuXVL//v3l6+urEiVKaOTIkTLGSJI+++wz1atXT97e3goMDFTnzp11/Phx+7l//fWXnnrqKfn7+6to0aKqVKmSZs6caT9+6NAhtW/fXn5+fipWrJhat26tAwcOZDuWqx87DA0N1YQJE/TMM8/I29tbZcqU0X//+1+Hc260DwAAAMAKhK8CaMnORD37v61KTEp1KD+alKpn/7f1lgew2bNny8XFRZs2bdK7776ryZMn6+OPP5YkXbx4UePGjdP27dv17bff6sCBA+revbv93JEjR2r37t368ccfFRsbq+nTp6tEiRL2cyMjI+Xt7a01a9Zo3bp18vLyUosWLZSWlpbj8b399tuqV6+etm3bpueee07PPvus4uLi8rQPAAAAIK/x2GEBk55hNPb73TJZHDOSbJLGfr9bD1UPvGWPIIaEhOidd96RzWZTlSpV9Ntvv+mdd95R79699cwzz9jrlS9fXu+9957q16+vc+fOycvLSwcPHtS9996revXqSbp8p+qKefPmKSMjQx9//LH9+5dmzpwpPz8/rVq1Sg8//HCOxteqVSs999xzkqRXXnlF77zzjlauXKkqVarkWR8AAABAXuPOVwGzKeF0pjte/2QkJSalalPC6Vs2hvvuu8/hy2nDw8O1d+9epaenKzo6WlFRUSpTpoy8vb3VtGlTSdLBgwclSc8++6y++OIL1a5dWy+//LJ+/fVXezvbt2/Xvn375O3tLS8vL3l5ealYsWJKTU1VfHx8jsdXs2ZN+59tNpsCAwPtjz7mVR8AAAAowDLSpYQ10m9fXf5vRnp+jyhHuPNVwBw/m33wyk29vJSamqrIyEhFRkZqzpw58vf318GDBxUZGWl/pK9ly5b6448/tHjxYv38889q3ry5+vXrp7feekvnzp1T3bp1NWfOnExt+/v753gcV+9+aLPZlJGRIUl51gcAAAAKqN3fSUtekZL//L8yn2CpxRtS9cfyb1w5QPgqYEp6u+dpvdzYuHGjw/sNGzaoUqVK2rNnj06dOqXXX39dISEhkqQtW7ZkOt/f31/dunVTt27d1LhxY7300kt66623VKdOHc2bN08lS5aUj4/PLRm7FX0AAAAgn+z+Tvqyq3T1h3SSEy+Xt/+0QAcwHjssYBqUK6YgX3dl92kum6Qg38vbzt8qBw8e1JAhQxQXF6fPP/9c77//vgYOHKgyZcrI1dVV77//vvbv36/vvvtO48aNczh31KhRWrhwofbt26ddu3bphx9+ULVq1SRJTz31lEqUKKHWrVtrzZo1SkhI0KpVq/T888/r8OHDeTJ2K/oAAABAPshIv3zHK9vdESQtGVqgH0EkfBUwzk42jY6qLkmZAtiV96Ojqt/S7/vq2rWr/v77bzVo0ED9+vXTwIED1adPH/n7+2vWrFmaP3++qlevrtdff11vvfWWw7murq4aNmyYatasqSZNmsjZ2VlffPGFJMnDw0O//PKLypQpo7Zt26patWrq2bOnUlNT8+wulRV9AAAAIB/88avjo4aZGCn5yOV6BZTNXPkCJ9yQ5ORk+fr6KikpKdNf6lNTU5WQkKBy5crJ3T13jwcu2Zmosd/vdth8I8jXXaOjqqvFPUE3NXbgTpEXv4sAAKCA+O0r6eue16/XboZU44lbP55/uFY2+Cc+81VAtbgnSA9VD9SmhNM6fjZVJb0vP2p4K+94AQAAAAWWV0De1ssHhK8CzNnJpvAKxfN7GAAAAED+K3v/5V0NkxOV9ee+bJePl73f6pHlGJ/5AgAAAFDwOTlf3k5eUra7I7R4/XK9AorwBQAAAKBwqP7Y5e3kfa7aA8EnuMBvMy/x2CEAAACAwqT6Y1LVRy7vanju2OXPeJW9v0Df8bqC8AUAAACgcHFylso1zu9R3DAeOwQAAAAACxC+AAAAAMAChC8AAAAAsADhC9e1bt061ahRQ0WKFFGbNm2uWXfVqlWy2Ww6c+bMTfXZrFkzDRo06KbaQOGRVz83AAAABRnhCw6yCj1DhgxR7dq1lZCQoFmzZuXLuAqaMWPGqHbt2vk9DAAAABQihK+CLCNdSlgj/fbV5f9mpOfLMOLj4/Xggw+qdOnS8vPzy5cxANeTlpaW30MAAAC4JsJXQbX7O2nKPdLsR6Wve17+75R7LpffIt27d9fq1av17rvvymaz2V+nTp3SM888I5vNluM7X9HR0apXr548PDx0//33Ky4uzqGfqx9fHDRokJo1a+ZQdunSJfXv31++vr4qUaKERo4cKWNMjvqfNm2aKlWqJHd3dwUEBOiJJ56wH8vIyNDEiRNVrlw5FS1aVLVq1dJXX31lP37lEbjly5dnOYdZs2Zp7Nix2r59u/0aXbkuZ86cUa9eveTv7y8fHx89+OCD2r59u73tK3fMPvvsM4WGhsrX11cdO3bU2bNnHcY3adIkVaxYUW5ubipTpozGjx9vP37o0CG1b99efn5+KlasmFq3bq0DBw7k6LpI0scff6xq1arJ3d1dVatW1bRp0+zHDhw4IJvNpgULFuiBBx6Qh4eHatWqpfXr1zu0sW7dOjVr1kweHh666667FBkZqb/++kuSdOHCBT3//PMqWbKk3N3d1ahRI23evNnh/MWLF6ty5coqWrSoHnjggSzHv3btWjVu3FhFixZVSEiInn/+eaWkpNiPh4aGaty4ceratat8fHzUp0+fHF8DAACA/ED4Koh2fyd92VVK/tOxPDnxcvktCmDvvvuuwsPD1bt3byUmJurw4cM6fPiwfHx8NGXKFCUmJqpDhw45amv48OF6++23tWXLFrm4uOiZZ5654fHMnj1bLi4u2rRpk959911NnjxZH3/88XXP27Jli55//nm99tpriouL05IlS9SkSRP78YkTJ+rTTz/Vhx9+qF27dmnw4MF6+umntXr16hzNoUOHDnrhhRd09913KzEx0eG6PPnkkzp+/Lh+/PFHRUdHq06dOmrevLlOnz5tbzc+Pl7ffvutfvjhB/3www9avXq1Xn/9dfvxYcOG6fXXX9fIkSO1e/duzZ07VwEBAZKkixcvKjIyUt7e3lqzZo3WrVsnLy8vtWjRIkd3fubMmaNRo0Zp/Pjxio2N1YQJEzRy5EjNnj0709xffPFFxcTEqHLlyurUqZMuXbokSYqJiVHz5s1VvXp1rV+/XmvXrlVUVJTS0y/fmX355Zf19ddfa/bs2dq6dasqVqyoyMhI+zU4dOiQ2rZtq6ioKMXExKhXr14aOnSoQ//x8fFq0aKF2rVrpx07dmjevHlau3at+vfv71DvrbfeUq1atbRt2zaNHDnyuvMHAADIVwa5kpSUZCSZpKSkTMf+/vtvs3v3bvP333/feMPpl4x5u6oxo32yefka83a1y/VugaZNm5qBAwc6lPn6+pqZM2fm6PyVK1caSWbZsmX2skWLFhlJ9uvRrVs307p1a4fzBg4caJo2beowjmrVqpmMjAx72SuvvGKqVat23TF8/fXXxsfHxyQnJ2c6lpqaajw8PMyvv/7qUN6zZ0/TqVOnHM9h9OjRplatWg5trFmzxvj4+JjU1FSH8goVKpj//Oc/9vM8PDwcxvbSSy+ZsLAwY4wxycnJxs3NzXz00UdZzu2zzz4zVapUcbguFy5cMEWLFjU//fTTNa/LlbHMnTvXoWzcuHEmPDzcGGNMQkKCkWQ+/vhj+/Fdu3YZSSY2NtYYY0ynTp1Mw4YNs2z/3LlzpkiRImbOnDn2srS0NBMcHGwmTZpkjDFm2LBhpnr16g7nvfLKK0aS+euvv4wxl9ejT58+DnXWrFljnJyc7GtQtmxZ06ZNm2vO96Z+FwEAAHLoWtngn1zyLfUha3/8mvmOlwMjJR+5XK8Af6t3zZo17X8OCgqSJB0/flxlypTJcRv33XefbDab/X14eLjefvttpaeny9nZOdvzHnroIZUtW1bly5dXixYt1KJFCz3++OPy8PDQvn37dP78eT300EMO56Slpenee++9qTls375d586dU/HixR3K//77b8XHx9vfh4aGytvb26Ht48ePS5JiY2N14cIFNW/ePNs+9u3b53C+JKWmpjr0kZWUlBTFx8erZ8+e6t27t7380qVL8vX1daib3dyrVq2qmJgYPfnkk1n2ER8fr4sXL6phw4b2siJFiqhBgwaKjY21zzEsLMzhvPDw8Ezz3LFjh+bMmWMvM8YoIyNDCQkJqlatmiSpXr1615wzAABAQUL4KmjOHcvbevmkSJEi9j9fCVAZGRmSJCcnp0yf3bp48WKe9e3t7a2tW7dq1apVWrp0qUaNGqUxY8Zo8+bNOnfunCRp0aJFKlWqlMN5bm5uOZ5DVs6dO6egoCCtWrUq07F/blTyz3avtH2l3aJFi15zbufOnVPdunUdQskV/v7+1z1Xkj766KNM4efqMHutuV9vjHnh3Llz+te//qXnn38+07F/hl9PT89bPhYAAIC8QvgqaLwC8rbeDXJ1dbV/dudW8ff3186dOx3KYmJiMoWSjRs3OrzfsGGDKlWqdM27Xle4uLgoIiJCERERGj16tPz8/LRixQo99NBDcnNz08GDB9W0adNczyGr61SnTh0dPXpULi4uCg0NzVW7lSpVUtGiRbV8+XL16tUr0/E6depo3rx5KlmypHx8fG6o7YCAAAUHB2v//v166qmncjU+6fJdseXLl2vs2LGZjlWoUEGurq5at26dypYtK+lysN68ebP9KwyqVaum775z/Nzihg0bHN7XqVNHu3fvVsWKFXM9TgAAgIKGDTcKmrL3Sz7BkmzZVLBJPqUu17sFQkNDtXHjRh04cEAnT5685p2e3HrwwQe1ZcsWffrpp9q7d69Gjx6dKYxJ0sGDBzVkyBDFxcXp888/1/vvv6+BAwdet/0ffvhB7733nmJiYvTHH3/o008/VUZGhqpUqSJvb2+9+OKLGjx4sGbPnq34+Hht3bpV77//fqZNJ64lNDRUCQkJiomJ0cmTJ3XhwgVFREQoPDxcbdq00dKlS3XgwAH9+uuvGj58uLZs2ZKjdt3d3fXKK6/o5Zdf1qeffqr4+Hht2LBBM2bMkCQ99dRTKlGihFq3bq01a9YoISFBq1at0vPPP6/Dhw9ft/2xY8dq4sSJeu+99/T777/rt99+08yZMzV58uQcz33YsGHavHmznnvuOe3YsUN79uzR9OnTdfLkSXl6eurZZ5/VSy+9pCVLlmj37t3q3bu3zp8/r549e0qS+vbtq7179+qll15SXFyc5s6dm2kXzVdeeUW//vqr+vfvr5iYGO3du1cLFy7MtOEGAABAYUL4KmicnKUWb/z/N1cHsP//vsXrl+vdAi+++KKcnZ1VvXp1+fv76+DBg3neR2RkpEaOHKmXX35Z9evX19mzZ9W1a9dM9bp27aq///5bDRo0UL9+/TRw4MAcbSfu5+enBQsW6MEHH1S1atX04Ycf6vPPP9fdd98tSRo3bpxGjhypiRMnqlq1amrRooUWLVqkcuXK5XgO7dq1U4sWLfTAAw/I399fn3/+uWw2mxYvXqwmTZqoR48eqly5sjp27Kg//vjDvlthTowcOVIvvPCCRo0apWrVqqlDhw72z4R5eHjol19+UZkyZdS2bVtVq1ZNPXv2VGpqao7uhPXq1Usff/yxZs6cqRo1aqhp06aaNWvWDc29cuXKWrp0qbZv364GDRooPDxcCxculIvL5Rvpr7/+utq1a6cuXbqoTp062rdvn3766Sfdddddki4/Nvj111/r22+/Va1atfThhx9qwoQJDn3UrFlTq1ev1u+//67GjRvr3nvv1ahRoxQcHJzjcQIAABQ0NnP1h2+QI8nJyfL19VVSUlKmv/SmpqYqISFB5cqVk7u7e+462P2dtOQVx803fEpdDl7VH7uJkQN3jjz5XQQAALiOa2WDf+IzXwVV9cekqo9c3tXw3LHLn/Eqe/8tu+MFAAAA4NbiscOCzMn58nbyNZ64/N8CELz69u0rLy+vLF99+/a1ZAxr1qzJdgxeXl6WjKGgutZ1WbNmTX4PDwAA4I7GnS/ckNdee00vvvhilsdudPe93KpXr55iYmIs6auwudZ1uXprfQAAAFiL8IUbUrJkSZUsWTJfx1C0aFG2IM8G1wUAAKDg4rHDW4i9TID8xe8gAAAoSAhft8CVLws+f/58Po8EuLOlpaVJUo6+mBsAAOBW47HDW8DZ2Vl+fn4O381ks2X3pckAboWMjAydOHFCHh4e9u8gAwAAyE/8jeQWCQwMlCR7AANgPScnJ5UpU4Z//AAAAAUC4esWsdlsCgoKUsmSJXXx4sX8Hg5wR3J1dZWTE09XAwCAgoHwdYs5OzvzeRMAAAAAbLgBAAAAAFYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCkT4mjp1qkJDQ+Xu7q6wsDBt2rTpmvWnTJmiKlWqqGjRogoJCdHgwYOVmppqPz5x4kTVr19f3t7eKlmypNq0aaO4uDiHNlJTU9WvXz8VL15cXl5eateunY4dO3ZL5gcAAAAA+R6+5s2bpyFDhmj06NHaunWratWqpcjISB0/fjzL+nPnztXQoUM1evRoxcbGasaMGZo3b55effVVe53Vq1erX79+2rBhg37++WddvHhRDz/8sFJSUux1Bg8erO+//17z58/X6tWr9eeff6pt27a3fL4AAAAA7kw2Y4zJzwGEhYWpfv36+uCDDyRJGRkZCgkJ0YABAzR06NBM9fv376/Y2FgtX77cXvbCCy9o48aNWrt2bZZ9nDhxQiVLltTq1avVpEkTJSUlyd/fX3PnztUTTzwhSdqzZ4+qVaum9evX67777rvuuJOTk+Xr66ukpCT5+PjkZuoAAAAAbgM5zQb5eucrLS1N0dHRioiIsJc5OTkpIiJC69evz/Kc+++/X9HR0fZHE/fv36/FixerVatW2faTlJQkSSpWrJgkKTo6WhcvXnTot2rVqipTpky2/V64cEHJyckOLwAAAADIKZf87PzkyZNKT09XQECAQ3lAQID27NmT5TmdO3fWyZMn1ahRIxljdOnSJfXt29fhscN/ysjI0KBBg9SwYUPdc889kqSjR4/K1dVVfn5+mfo9evRolu1MnDhRY8eOvcEZAgAAAMBl+f6Zrxu1atUqTZgwQdOmTdPWrVu1YMECLVq0SOPGjcuyfr9+/bRz50598cUXN9XvsGHDlJSUZH8dOnToptoDAAAAcGfJ1ztfJUqUkLOzc6ZdBo8dO6bAwMAszxk5cqS6dOmiXr16SZJq1KihlJQU9enTR8OHD5eT0//lyf79++uHH37QL7/8otKlS9vLAwMDlZaWpjNnzjjc/bpWv25ubnJzc8vtVAEAAADc4fL1zperq6vq1q3rsHlGRkaGli9frvDw8CzPOX/+vEPAkiRnZ2dJ0pW9Q4wx6t+/v7755hutWLFC5cqVc6hft25dFSlSxKHfuLg4HTx4MNt+AQAAAOBm5OudL0kaMmSIunXrpnr16qlBgwaaMmWKUlJS1KNHD0lS165dVapUKU2cOFGSFBUVpcmTJ+vee+9VWFiY9u3bp5EjRyoqKsoewvr166e5c+dq4cKF8vb2tn+Oy9fXV0WLFpWvr6969uypIUOGqFixYvLx8dGAAQMUHh6eo50OAQAAAOBG5Xv46tChg06cOKFRo0bp6NGjql27tpYsWWLfhOPgwYMOd7pGjBghm82mESNG6MiRI/L391dUVJTGjx9vrzN9+nRJUrNmzRz6mjlzprp37y5Jeuedd+Tk5KR27drpwoULioyM1LRp027tZAEAAADcsfL9e74KK77nCwAAAIBUSL7nCwAAAADuFIQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAvke/iaOnWqQkND5e7urrCwMG3atOma9adMmaIqVaqoaNGiCgkJ0eDBg5Wammo//ssvvygqKkrBwcGy2Wz69ttvM7XRvXt32Ww2h1eLFi3yemoAAAAAYJev4WvevHkaMmSIRo8era1bt6pWrVqKjIzU8ePHs6w/d+5cDR06VKNHj1ZsbKxmzJihefPm6dVXX7XXSUlJUa1atTR16tRr9t2iRQslJibaX59//nmezg0AAAAA/sklPzufPHmyevfurR49ekiSPvzwQy1atEiffPKJhg4dmqn+r7/+qoYNG6pz586SpNDQUHXq1EkbN26012nZsqVatmx53b7d3NwUGBiYRzMBAAAAgGvLtztfaWlpio6OVkRExP8NxslJERERWr9+fZbn3H///YqOjrY/mrh//34tXrxYrVq1uuH+V61apZIlS6pKlSp69tlnderUqWvWv3DhgpKTkx1eAAAAAJBT+Xbn6+TJk0pPT1dAQIBDeUBAgPbs2ZPlOZ07d9bJkyfVqFEjGWN06dIl9e3b1+Gxw5xo0aKF2rZtq3Llyik+Pl6vvvqqWrZsqfXr18vZ2TnLcyZOnKixY8feUD8AAAAAcEW+b7hxI1atWqUJEyZo2rRp2rp1qxYsWKBFixZp3LhxN9ROx44d9dhjj6lGjRpq06aNfvjhB23evFmrVq3K9pxhw4YpKSnJ/jp06NBNzgYAAADAnSTf7nyVKFFCzs7OOnbsmEP5sWPHsv0s1siRI9WlSxf16tVLklSjRg2lpKSoT58+Gj58uJyccpcly5cvrxIlSmjfvn1q3rx5lnXc3Nzk5uaWq/YBAAAAIN/ufLm6uqpu3bpavny5vSwjI0PLly9XeHh4luecP38+U8C68pigMSbXYzl8+LBOnTqloKCgXLcBAAAAANeSr7sdDhkyRN26dVO9evXUoEEDTZkyRSkpKfbdD7t27apSpUpp4sSJkqSoqChNnjxZ9957r8LCwrRv3z6NHDlSUVFR9hB27tw57du3z95HQkKCYmJiVKxYMZUpU0bnzp3T2LFj1a5dOwUGBio+Pl4vv/yyKlasqMjISOsvAgAAAIA7Qr6Grw4dOujEiRMaNWqUjh49qtq1a2vJkiX2TTgOHjzocKdrxIgRstlsGjFihI4cOSJ/f39FRUVp/Pjx9jpbtmzRAw88YH8/ZMgQSVK3bt00a9YsOTs7a8eOHZo9e7bOnDmj4OBgPfzwwxo3bhyPFQIAAAC4ZWzmZp7Xu4MlJyfL19dXSUlJ8vHxye/hAAAAAMgnOc0GhWq3QwAAAAAorAhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABfI9fE2dOlWhoaFyd3dXWFiYNm3adM36U6ZMUZUqVVS0aFGFhIRo8ODBSk1NtR//5ZdfFBUVpeDgYNlsNn377beZ2jDGaNSoUQoKClLRokUVERGhvXv35vXUAAAAAMAuX8PXvHnzNGTIEI0ePVpbt25VrVq1FBkZqePHj2dZf+7cuRo6dKhGjx6t2NhYzZgxQ/PmzdOrr75qr5OSkqJatWpp6tSp2fY7adIkvffee/rwww+1ceNGeXp6KjIy0iHEAQAAAEBeshljTH51HhYWpvr16+uDDz6QJGVkZCgkJEQDBgzQ0KFDM9Xv37+/YmNjtXz5cnvZCy+8oI0bN2rt2rWZ6ttsNn3zzTdq06aNvcwYo+DgYL3wwgt68cUXJUlJSUkKCAjQrFmz1LFjxyzHeuHCBV24cMH+Pjk5WSEhIUpKSpKPj0+u5g8AAACg8EtOTpavr+91s0G+3flKS0tTdHS0IiIi/m8wTk6KiIjQ+vXrszzn/vvvV3R0tP3RxP3792vx4sVq1apVjvtNSEjQ0aNHHfr19fVVWFhYtv1K0sSJE+Xr62t/hYSE5LhPAAAAAHDJr45Pnjyp9PR0BQQEOJQHBARoz549WZ7TuXNnnTx5Uo0aNZIxRpcuXVLfvn0dHju8nqNHj9r7ubrfK8eyMmzYMA0ZMsT+/sqdLwAAAADIiXzfcONGrFq1ShMmTNC0adO0detWLViwQIsWLdK4ceNued9ubm7y8fFxeAEAAABATuXbna8SJUrI2dlZx44dcyg/duyYAgMDszxn5MiR6tKli3r16iVJqlGjhlJSUtSnTx8NHz5cTk7Xz5JX2j527JiCgoIc+q1du3YuZwMAAAAA15Zvd75cXV1Vt25dh80zMjIytHz5coWHh2d5zvnz5zMFLGdnZ0mXN9LIiXLlyikwMNCh3+TkZG3cuDHbfgEAAADgZuXbnS9JGjJkiLp166Z69eqpQYMGmjJlilJSUtSjRw9JUteuXVWqVClNnDhRkhQVFaXJkyfr3nvvVVhYmPbt26eRI0cqKirKHsLOnTunffv22ftISEhQTEyMihUrpjJlyshms2nQoEH697//rUqVKqlcuXIaOXKkgoODHXZFBAAAAIC8lK/hq0OHDjpx4oRGjRqlo0ePqnbt2lqyZIl9M4yDBw863OkaMWKEbDabRowYoSNHjsjf319RUVEaP368vc6WLVv0wAMP2N9f2SSjW7dumjVrliTp5Zdftj+ueObMGTVq1EhLliyRu7u7BbMGAAAAcCe6qe/52rdvn+Lj49WkSRMVLVpUxhjZbLa8HF+BldO9/AEAAADc3m7p93ydOnVKERERqly5slq1aqXExERJUs+ePfXCCy/kbsQAAAAAcBvLVfgaPHiwXFxcdPDgQXl4eNjLO3TooCVLluTZ4AAAAADgdpGrz3wtXbpUP/30k0qXLu1QXqlSJf3xxx95MjAAAAAAuJ3k6s5XSkqKwx2vK06fPi03N7ebHhQAAAAA3G5yFb4aN26sTz/91P7eZrMpIyNDkyZNcthpEAAAAABwWa4eO5w0aZKaN2+uLVu2KC0tTS+//LJ27dql06dPa926dXk9RgAAAAAo9HJ15+uee+7R77//rkaNGql169ZKSUlR27ZttW3bNlWoUCGvxwgAAAAAhd4N3/m6ePGiWrRooQ8//FDDhw+/FWMCAAAAgNvODd/5KlKkiHbs2HErxgIAAAAAt61cPXb49NNPa8aMGXk9FgAAAAC4beVqw41Lly7pk08+0bJly1S3bl15eno6HJ88eXKeDA4AAAAAbhe5Cl87d+5UnTp1JEm///67wzGbzXbzowIAAACA20yuwtfKlSvzehwAAAAAcFvL1We+/unw4cM6fPhwXowFAAAAAG5buQpfGRkZeu211+Tr66uyZcuqbNmy8vPz07hx45SRkZHXYwQAAACAQi9Xjx0OHz5cM2bM0Ouvv66GDRtKktauXasxY8YoNTVV48ePz9NBAgAAAEBhZzPGmBs9KTg4WB9++KEee+wxh/KFCxfqueee05EjR/JsgAVVcnKyfH19lZSUJB8fn/weDgAAAIB8ktNskKvHDk+fPq2qVatmKq9atapOnz6dmyYBAAAA4LaWq/BVq1YtffDBB5nKP/jgA9WqVeumBwUAAAAAt5tcfeZr0qRJeuSRR7Rs2TKFh4dLktavX69Dhw5p8eLFeTpAAAAAALgd5OrOV9OmTRUXF6fHH39cZ86c0ZkzZ9S2bVvFxcWpcePGeT1GAAAAACj0crXhBthwAwAAAMBlt3TDjZkzZ2r+/PmZyufPn6/Zs2fnpkkAAAAAuK3lKnxNnDhRJUqUyFResmRJTZgw4aYHBQAAAAC3m1yFr4MHD6pcuXKZysuWLauDBw/e9KAAAAAA4HaTq/BVsmRJ7dixI1P59u3bVbx48ZseFAAAAADcbnIVvjp16qTnn39eK1euVHp6utLT07VixQoNHDhQHTt2zOsxAgAAAEChl6vv+Ro3bpwOHDig5s2by8XlchMZGRnq2rUrn/kCAAAAgCzc1Fbze/fuVUxMjIoWLaoaNWqobNmyeTm2Ao2t5gEAAABIOc8GubrzdUWlSpVUqVIlpaen67fffpOPj4/uuuuum2kSAAAAAG5LufrM16BBgzRjxgxJUnp6upo2bao6deooJCREq1atysvxAQAAAMBtIVfh66uvvlKtWrUkSd9//73279+vPXv2aPDgwRo+fHieDhAAAAAAbge5Cl8nT55UYGCgJGnx4sVq3769KleurGeeeUa//fZbng4QAAAAAG4HuQpfAQEB2r17t9LT07VkyRI99NBDkqTz58/L2dk5TwcIAAAAALeDXG240aNHD7Vv315BQUGy2WyKiIiQJG3cuFFVq1bN0wECAAAAwO0gV+FrzJgxuueee3To0CE9+eSTcnNzkyQ5Oztr6NCheTpAAAAAALgd3NT3fEnS4cOHFRwcLCenXD3BWGjxPV8AAAAApJxng5tOTNWrV9eBAwduthkAAAAAuK3ddPi6yRtnAAAAAHBHuLOeFQQAAACAfHLT4evVV19VsWLF8mIsAAAAAHDbuukNN+5UbLgBAAAAQLJww41/OnTokJ555pm8bBIAAAAAbgt5Gr5Onz6t2bNn52WTAAAAAHBbuKEvWf7uu++ueXz//v03NRgAAAAAuF3dUPhq06aNbDbbNbeXt9lsNz0oAAAAALjd3NBjh0FBQVqwYIEyMjKyfG3duvVWjRMAAAAACrUbCl9169ZVdHR0tsevd1cMAAAAAO5UN/TY4UsvvaSUlJRsj1esWFErV6686UEBAAAAwO3mhsJXqVKlVK5cuWyPe3p6qmnTpjc9KAAAAAC43dzQY4eVKlXSiRMn7O87dOigY8eO5fmgAAAAAOB2c0Ph6+rPcy1evPiajyECAAAAAC7L0y9ZBgAAAABk7YbCl81my/Q9XnyvFwAAAABc3w1tuGGMUffu3eXm5iZJSk1NVd++feXp6elQb8GCBXk3QgAAAAC4DdxQ+OrWrZvD+6effjpPBwMAAAAAt6sbCl8zZ868VeMAAAAAgNsaG24AAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFigQISvqVOnKjQ0VO7u7goLC9OmTZuuWX/KlCmqUqWKihYtqpCQEA0ePFipqak31GazZs1ks9kcXn379s3zuQEAAACAVADC17x58zRkyBCNHj1aW7duVa1atRQZGanjx49nWX/u3LkaOnSoRo8erdjYWM2YMUPz5s3Tq6++esNt9u7dW4mJifbXpEmTbulcAQAAANy58j18TZ48Wb1791aPHj1UvXp1ffjhh/Lw8NAnn3ySZf1ff/1VDRs2VOfOnRUaGqqHH35YnTp1crizldM2PTw8FBgYaH/5+Pjc0rkCAAAAuHPla/hKS0tTdHS0IiIi7GVOTk6KiIjQ+vXrszzn/vvvV3R0tD1s7d+/X4sXL1arVq1uuM05c+aoRIkSuueeezRs2DCdP38+27FeuHBBycnJDi8AAAAAyCmX/Oz85MmTSk9PV0BAgEN5QECA9uzZk+U5nTt31smTJ9WoUSMZY3Tp0iX17dvX/thhTtvs3LmzypYtq+DgYO3YsUOvvPKK4uLitGDBgiz7nThxosaOHXsz0wUAAABwB8vX8JUbq1at0oQJEzRt2jSFhYVp3759GjhwoMaNG6eRI0fmuJ0+ffrY/1yjRg0FBQWpefPmio+PV4UKFTLVHzZsmIYMGWJ/n5ycrJCQkJubDAAAAIA7Rr6GrxIlSsjZ2VnHjh1zKD927JgCAwOzPGfkyJHq0qWLevXqJelycEpJSVGfPn00fPjwXLUpSWFhYZKkffv2ZRm+3Nzc5ObmdkPzAwAAAIAr8vUzX66urqpbt66WL19uL8vIyNDy5csVHh6e5Tnnz5+Xk5PjsJ2dnSVJxphctSlJMTExkqSgoKDcTgcAAAAAspXvjx0OGTJE3bp1U7169dSgQQNNmTJFKSkp6tGjhySpa9euKlWqlCZOnChJioqK0uTJk3XvvffaHzscOXKkoqKi7CHsem3Gx8dr7ty5atWqlYoXL64dO3Zo8ODBatKkiWrWrJk/FwIAAADAbS3fw1eHDh104sQJjRo1SkePHlXt2rW1ZMkS+4YZBw8edLjTNWLECNlsNo0YMUJHjhyRv7+/oqKiNH78+By36erqqmXLltlDWUhIiNq1a6cRI0ZYO3kAAAAAdwybMcbk9yAKo+TkZPn6+iopKYnvBwMAAADuYDnNBvn+JcsAAAAAcCcgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYoECEr6lTpyo0NFTu7u4KCwvTpk2brll/ypQpqlKliooWLaqQkBANHjxYqampN9Rmamqq+vXrp+LFi8vLy0vt2rXTsWPH8nxuAAAAACAVgPA1b948DRkyRKNHj9bWrVtVq1YtRUZG6vjx41nWnzt3roYOHarRo0crNjZWM2bM0Lx58/Tqq6/eUJuDBw/W999/r/nz52v16tX6888/1bZt21s+XwAAAAB3JpsxxuTnAMLCwlS/fn198MEHkqSMjAyFhIRowIABGjp0aKb6/fv3V2xsrJYvX24ve+GFF7Rx40atXbs2R20mJSXJ399fc+fO1RNPPCFJ2rNnj6pVq6b169frvvvuu+64k5OT5evrq6SkJPn4+Nz0dQAAAABQOOU0G+Trna+0tDRFR0crIiLCXubk5KSIiAitX78+y3Puv/9+RUdH2x8j3L9/vxYvXqxWrVrluM3o6GhdvHjRoU7VqlVVpkyZbPu9cOGCkpOTHV4AAAAAkFMu+dn5yZMnlZ6eroCAAIfygIAA7dmzJ8tzOnfurJMnT6pRo0YyxujSpUvq27ev/bHDnLR59OhRubq6ys/PL1Odo0ePZtnvxIkTNXbs2NxMEwAAAADy/zNfN2rVqlWaMGGCpk2bpq1bt2rBggVatGiRxo0bd0v7HTZsmJKSkuyvQ4cO3dL+AAAAANxe8vXOV4kSJeTs7Jxpl8Fjx44pMDAwy3NGjhypLl26qFevXpKkGjVqKCUlRX369NHw4cNz1GZgYKDS0tJ05swZh7tf1+rXzc1Nbm5uuZ0qAAAAgDtcvt75cnV1Vd26dR02z8jIyNDy5csVHh6e5Tnnz5+Xk5PjsJ2dnSVJxpgctVm3bl0VKVLEoU5cXJwOHjyYbb8AAAAAcDPy9c6XJA0ZMkTdunVTvXr11KBBA02ZMkUpKSnq0aOHJKlr164qVaqUJk6cKEmKiorS5MmTde+99yosLEz79u3TyJEjFRUVZQ9h12vT19dXPXv21JAhQ1SsWDH5+PhowIABCg8Pz9FOhwAAAABwo/I9fHXo0EEnTpzQqFGjdPToUdWuXVtLliyxb5hx8OBBhztdI0aMkM1m04gRI3TkyBH5+/srKipK48ePz3GbkvTOO+/IyclJ7dq104ULFxQZGalp06ZZN3EAAAAAd5R8/56vworv+QIAAAAgFZLv+QIAAACAOwXhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMACBSJ8TZ06VaGhoXJ3d1dYWJg2bdqUbd1mzZrJZrNlej3yyCP2OseOHVP37t0VHBwsDw8PtWjRQnv37r1uO3379r1lcwQAAABwZ8v38DVv3jwNGTJEo0eP1tatW1WrVi1FRkbq+PHjWdZfsGCBEhMT7a+dO3fK2dlZTz75pCTJGKM2bdpo//79WrhwobZt26ayZcsqIiJCKSkpDm317t3boa1Jkybd8vkCAAAAuDPle/iaPHmyevfurR49eqh69er68MMP5eHhoU8++STL+sWKFVNgYKD99fPPP8vDw8Mevvbu3asNGzZo+vTpql+/vqpUqaLp06fr77//1ueff+7QloeHh0NbPj4+t3y+AAAAAO5M+Rq+0tLSFB0drYiICHuZk5OTIiIitH79+hy1MWPGDHXs2FGenp6SpAsXLkiS3N3dHdp0c3PT2rVrHc6dM2eOSpQooXvuuUfDhg3T+fPns+3nwoULSk5OdngBAAAAQE7la/g6efKk0tPTFRAQ4FAeEBCgo0ePXvf8TZs2aefOnerVq5e9rGrVqipTpoyGDRumv/76S2lpaXrjjTd0+PBhJSYm2ut17txZ//vf/7Ry5UoNGzZMn332mZ5++uls+5o4caJ8fX3tr5CQkFzMGAAAAMCdyiW/B3AzZsyYoRo1aqhBgwb2siJFimjBggXq2bOnihUrJmdnZ0VERKhly5Yyxtjr9enTx/7nGjVqKCgoSM2bN1d8fLwqVKiQqa9hw4ZpyJAh9vfJyckEMAAAAAA5lq/hq0SJEnJ2dtaxY8ccyo8dO6bAwMBrnpuSkqIvvvhCr732WqZjdevWVUxMjJKSkpSWliZ/f3+FhYWpXr162bYXFhYmSdq3b1+W4cvNzU1ubm45mRYAAAAAZJKvjx26urqqbt26Wr58ub0sIyNDy5cvV3h4+DXPnT9/vi5cuHDNRwV9fX3l7++vvXv3asuWLWrdunW2dWNiYiRJQUFBNzYJAAAAAMiBfH/scMiQIerWrZvq1aunBg0aaMqUKUpJSVGPHj0kSV27dlWpUqU0ceJEh/NmzJihNm3aqHjx4pnanD9/vvz9/VWmTBn99ttvGjhwoNq0aaOHH35YkhQfH6+5c+eqVatWKl68uHbs2KHBgwerSZMmqlmz5q2fNAAAAIA7Tr6Hrw4dOujEiRMaNWqUjh49qtq1a2vJkiX2TTgOHjwoJyfHG3RxcXFau3atli5dmmWbiYmJGjJkiI4dO6agoCB17dpVI0eOtB93dXXVsmXL7EEvJCRE7dq104gRI27dRAEAAADc0Wzmn7tQIMeSk5Pl6+urpKQkvh8MAAAAuIPlNBvk+5csAwAAAMCdgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABl/weAG5OeobRpoTTOn42VSW93dWgXDE5O9nye1gAAAAArkL4KsSW7EzU2O93KzEp1V4W5Ouu0VHV1eKeoHwcGQAAAICr8dhhIbVkZ6Ke/d9Wh+AlSUeTUvXs/7Zqyc7EfBoZAAAAgKwQvgqh9Ayjsd/vlsni2JWysd/vVnpGVjUAAAAA5AfCVyG0KeF0pjte/2QkJSalalPCaesGBQAAAOCaCF+F0PGz2Qev3NQDAAAAcOsRvgqhkt7ueVoPAAAAwK1H+CqEGpQrpiBfd2W3obxNl3c9bFCumJXDAgAAAHANhK9CyNnJptFR1SUpUwC78n50VHW+7wsAAAAoQAhfhVSLe4I0/ek6CvR1fLQw0Ndd05+uw/d8AQAAAAUMX7JciLW4J0gPVQ/UpoTTOn42VSW9Lz9qyB0vAAAAoOAhfBVyzk42hVcont/DAAAAAHAdPHYIAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWcMnvARRWxhhJUnJycj6PBAAAAEB+upIJrmSE7BC+cuns2bOSpJCQkHweCQAAAICC4OzZs/L19c32uM1cL54hSxkZGfrzzz/l7e0tm82W38O54yUnJyskJESHDh2Sj49Pfg8H2WCdCgfWqeBjjQoH1qlwYJ0Kh4K+TsYYnT17VsHBwXJyyv6TXdz5yiUnJyeVLl06v4eBq/j4+BTIX0g4Yp0KB9ap4GONCgfWqXBgnQqHgrxO17rjdQUbbgAAAACABQhfAAAAAGABwhduC25ubho9erTc3Nzyeyi4BtapcGCdCj7WqHBgnQoH1qlwuF3WiQ03AAAAAMAC3PkCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4Qr6YOnWqQkND5e7urrCwMG3atOma9efPn6+qVavK3d1dNWrU0OLFix2OG2M0atQoBQUFqWjRooqIiNDevXsd6pw+fVpPPfWUfHx85Ofnp549e+rcuXP246mpqerevbtq1KghFxcXtWnTJs/mW1gVxHVatWqVWrduraCgIHl6eqp27dqaM2dO3k26kCmIaxQXF6cHHnhAAQEBcnd3V/ny5TVixAhdvHgx7yZeyBTEdfqnffv2ydvbW35+fjc1z8KuIK7TgQMHZLPZMr02bNiQdxMvZAriOl1p56233lLlypXl5uamUqVKafz48Xkz6UKoIK7TmDFjsvx98vT0zLuJX48BLPbFF18YV1dX88knn5hdu3aZ3r17Gz8/P3Ps2LEs669bt844OzubSZMmmd27d5sRI0aYIkWKmN9++81e5/XXXze+vr7m22+/Ndu3bzePPfaYKVeunPn777/tdVq0aGFq1aplNmzYYNasWWMqVqxoOnXqZD9+7tw507dvX/Pf//7XREZGmtatW9+ya1AYFNR1Gj9+vBkxYoRZt26d2bdvn5kyZYpxcnIy33///a27GAVUQV2j+Ph488knn5iYmBhz4MABs3DhQlOyZEkzbNiwW3cxCrCCuk5XpKWlmXr16pmWLVsaX1/fPJ9/YVFQ1ykhIcFIMsuWLTOJiYn2V1pa2q27GAVYQV0nY4wZMGCAqVKlilm4cKHZv3+/2bJli1m6dOmtuRAFXEFdp7Nnzzr8HiUmJprq1aubbt263bJrcTXCFyzXoEED069fP/v79PR0ExwcbCZOnJhl/fbt25tHHnnEoSwsLMz861//MsYYk5GRYQIDA82bb75pP37mzBnj5uZmPv/8c2OMMbt37zaSzObNm+11fvzxR2Oz2cyRI0cy9dmtW7c7PnwVhnW6olWrVqZHjx43PslCrjCt0eDBg02jRo1ufJK3gYK+Ti+//LJ5+umnzcyZM+/o8FVQ1+lK+Nq2bVuezLOwK6jrtHv3buPi4mL27NmTNxMt5ArqOl0tJibGSDK//PJL7iaaCzx2CEulpaUpOjpaERER9jInJydFRERo/fr1WZ6zfv16h/qSFBkZaa+fkJCgo0ePOtTx9fVVWFiYvc769evl5+enevXq2etERETIyclJGzduzLP53S4K2zolJSWpWLFiNz7RQqwwrdG+ffu0ZMkSNW3aNHeTLcQK+jqtWLFC8+fP19SpU29+soVYQV8nSXrsscdUsmRJNWrUSN99993NTbiQKsjr9P3336t8+fL64YcfVK5cOYWGhqpXr146ffp03ky+ECnI63S1jz/+WJUrV1bjxo1zN9lcIHzBUidPnlR6eroCAgIcygMCAnT06NEszzl69Og161/57/XqlCxZ0uG4i4uLihUrlm2/d7LCtE5ffvmlNm/erB49euRwdreHwrBG999/v9zd3VWpUiU1btxYr7322g3OsvAryOt06tQpde/eXbNmzZKPj08uZ3h7KMjr5OXlpbffflvz58/XokWL1KhRI7Vp0+aODGAFeZ3279+vP/74Q/Pnz9enn36qWbNmKTo6Wk888UQuZ1t4FeR1+qfU1FTNmTNHPXv2vIHZ3TwXS3sDgDy0cuVK9ejRQx999JHuvvvu/B4OrjJv3jydPXtW27dv10svvaS33npLL7/8cn4PC/9f79691blzZzVp0iS/h4JrKFGihIYMGWJ/X79+ff35559688039dhjj+XjyPBPGRkZunDhgj799FNVrlxZkjRjxgzVrVtXcXFxqlKlSj6PEFf75ptvdPbsWXXr1s3SfrnzBUuVKFFCzs7OOnbsmEP5sWPHFBgYmOU5gYGB16x/5b/Xq3P8+HGH45cuXdLp06ez7fdOVhjWafXq1YqKitI777yjrl273uAMC7/CsEYhISGqXr26OnXqpNdff11jxoxRenr6Dc60cCvI67RixQq99dZbcnFxkYuLi3r27KmkpCS5uLjok08+yeWMC6eCvE5ZCQsL0759+3Iws9tLQV6noKAgubi42IOXJFWrVk2SdPDgwRuaZ2FXkNfpnz7++GM9+uijme6m3WqEL1jK1dVVdevW1fLly+1lGRkZWr58ucLDw7M8Jzw83KG+JP3888/2+uXKlVNgYKBDneTkZG3cuNFeJzw8XGfOnFF0dLS9zooVK5SRkaGwsLA8m9/toqCv06pVq/TII4/ojTfeUJ8+fW5+woVQQV+jq2VkZOjixYvKyMi48ckWYgV5ndavX6+YmBj767XXXpO3t7diYmL0+OOP580FKCQK8jplJSYmRkFBQTc+0UKuIK9Tw4YNdenSJcXHx9vr/P7775KksmXL3sy0C52CvE5XJCQkaOXKlZY/ciiJreZhvS+++MK4ubmZWbNmmd27d5s+ffoYPz8/c/ToUWOMMV26dDFDhw6111+3bp1xcXExb731lomNjTWjR4/OcvtRPz8/s3DhQrNjxw7TunXrLLcfvffee83GjRvN2rVrTaVKlTJtE7tr1y6zbds2ExUVZZo1a2a2bdt2x+4wVVDXacWKFcbDw8MMGzbMYavYU6dOWXBVCpaCukb/+9//zLx588zu3btNfHy8mTdvngkODjZPPfWUBVel4Cmo63S1O323w4K6TrNmzTJz5841sbGxJjY21owfP944OTmZTz75xIKrUvAU1HVKT083derUMU2aNDFbt241W7ZsMWFhYeahhx6y4KoUPAV1na4YMWKECQ4ONpcuXbqFVyFrhC/ki/fff9+UKVPGuLq6mgYNGpgNGzbYjzVt2jTT9y18+eWXpnLlysbV1dXcfffdZtGiRQ7HMzIyzMiRI01AQIBxc3MzzZs3N3FxcQ51Tp06ZTp16mS8vLyMj4+P6dGjhzl79qxDnbJlyxpJmV53qoK4Tt26dctyjZo2bZrn8y8MCuIaffHFF6ZOnTrGy8vLeHp6murVq5sJEyY4/B/knaYgrtPV7vTwZUzBXKdZs2aZatWqGQ8PD+Pj42MaNGhg5s+fn/eTL0QK4joZY8yRI0dM27ZtjZeXlwkICDDdu3e/I/9h8IqCuk7p6emmdOnS5tVXX83bCeeQzRhjrL/fBgAAAAB3Fj7zBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFALeJ7t27q02bNvk9jNuOzWbTt99+K0k6cOCAbDabYmJict1eXrSRU2lpaapYsaJ+/fXXW97XrXQjP9tDhw7VgAEDbu2AACCXXPJ7AACA67PZbNc8Pnr0aL377rsyxlg0ojtTSEiIEhMTVaJEiRzV7969u86cOWMPb7lp42Z8+OGHKleunO6///5b3ldB8eKLL6p8+fIaPHiwypcvn9/DAQAH3PkCgEIgMTHR/poyZYp8fHwcyl588UX5+vrKz88vv4eaZy5evFjg2nJ2dlZgYKBcXHL/b5d50UZOGGP0wQcfqGfPnre0n4KmRIkSioyM1PTp0/N7KACQCeELAAqBwMBA+8vX11c2m82hzMvLK9OjWc2aNdOAAQM0aNAg3XXXXQoICNBHH32klJQU9ejRQ97e3qpYsaJ+/PFHh7527typli1bysvLSwEBAerSpYtOnjyZ7dhmzZolPz8/ffvtt6pUqZLc3d0VGRmpQ4cOOdRbuHCh6tSpI3d3d5UvX15jx47VpUuX7MdtNpumT5+uxx57TJ6enho/fnyW/YWGhmrcuHHq1KmTPD09VapUKU2dOtWhTnZtXW8Me/fuVZMmTeTu7q7q1avr559/dmg3q0cGd+3apUcffVQ+Pj7y9vZW48aNFR8frzFjxmj27NlauHChbDabbDabVq1alWUbq1evVoMGDeTm5qagoCANHTrUYVzNmjXT888/r5dfflnFihVTYGCgxowZk+2aSFJ0dLTi4+P1yCOP2MvS0tLUv39/BQUFyd3dXWXLltXEiRPtx8+cOaNevXrJ399fPj4+evDBB7V9+3aHdr///nvVr19f7u7uKlGihB5//HH7sb/++ktdu3bVXXfdJQ8PD7Vs2VJ79+61H7/ys/LTTz+pWrVq8vLyUosWLZSYmGivk56eriFDhsjPz0/FixfXyy+/nOmO7ldffaUaNWqoaNGiKl68uCIiIpSSkmI/HhUVpS+++OKa1wcA8gPhCwBuY7Nnz1aJEiW0adMmDRgwQM8++6yefPJJ3X///dq6dasefvhhdenSRefPn5d0+S/fDz74oO69915t2bJFS5Ys0bFjx9S+fftr9nP+/HmNHz9en376qdatW6czZ86oY8eO9uNr1qxR165dNXDgQO3evVv/+c9/NGvWrEwBa8yYMXr88cf122+/6Zlnnsm2vzfffFO1atXStm3bNHToUA0cODBTULq6reuNISMjQ23btpWrq6s2btyoDz/8UK+88so1533kyBE1adJEbm5uWrFihaKjo/XMM8/o0qVLevHFF9W+fXt7uEhMTMzy8b8jR46oVatWql+/vrZv367p06drxowZ+ve//+1Qb/bs2fL09NTGjRs1adIkvfbaa5nm/E9r1qxR5cqV5e3tbS9777339N133+nLL79UXFyc5syZo9DQUPvxJ598UsePH9ePP/6o6Oho1alTR82bN9fp06clSYsWLdLjjz+uVq1aadu2bVq+fLkaNGhgP7979+7asmWLvvvuO61fv17GGLVq1crhzuP58+f11ltv6bPPPtMvv/yigwcP6sUXX7Qff/vttzVr1ix98sknWrt2rU6fPq1vvvnGfjwxMVGdOnXSM888o9jYWK1atUpt27Z1CGgNGjTQ4cOHdeDAgWusHgDkAwMAKFRmzpxpfH19M5V369bNtG7d2v6+adOmplGjRvb3ly5dMp6enqZLly72ssTERCPJrF+/3hhjzLhx48zDDz/s0O6hQ4eMJBMXF5fteCSZDRs22MtiY2ONJLNx40ZjjDHNmzc3EyZMcDjvs88+M0FBQfb3ksygQYOuM3tjypYta1q0aOFQ1qFDB9OyZctrtnW9Mfz000/GxcXFHDlyxH78xx9/NJLMN998Y4wxJiEhwUgy27ZtM8YYM2zYMFOuXDmTlpaW5VivXpOs2nj11VdNlSpVTEZGhr3O1KlTjZeXl0lPTzfGZF5LY4ypX7++eeWVV7Ls1xhjBg4caB588EGHsgEDBpgHH3zQoa8r1qxZY3x8fExqaqpDeYUKFcx//vMfY4wx4eHh5qmnnsqyv99//91IMuvWrbOXnTx50hQtWtR8+eWXxpj/+1nZt2+fw1wDAgLs74OCgsykSZPs7y9evGhKly5tv47R0dFGkjlw4EC2c09KSjKSzKpVq7KtAwD5gTtfAHAbq1mzpv3Pzs7OKl68uGrUqGEvCwgIkCQdP35ckrR9+3atXLlSXl5e9lfVqlUlSfHx8dn24+Liovr169vfV61aVX5+foqNjbW3+9prrzm027t3byUmJtrvuklSvXr1cjSv8PDwTO+v9JVdW9cbQ2xsrEJCQhQcHJxtP1eLiYlR48aNVaRIkRyNOyuxsbEKDw932FSlYcOGOnfunA4fPmwv++daSlJQUJB93bLy999/y93d3aGse/fuiomJUZUqVfT8889r6dKl9mPbt2/XuXPnVLx4cYdrlJCQYF/7mJgYNW/ePNt5uLi4KCwszF5WvHhxValSxWFtPDw8VKFChSznkZSUpMTERIc2XFxcHNayVq1aat68uWrUqKEnn3xSH330kf766y+HsRQtWlSSHH62AKAgYLdDALiNXR0KbDabQ9mVv/BnZGRIks6dO6eoqCi98cYbmdoKCgrK9TjOnTunsWPHqm3btpmO/TMgeHp65rqPq13dVk7HcCOu/CXfClmt5ZV1y0qJEiX022+/OZTVqVNHCQkJ+vHHH7Vs2TK1b99eERER+uqrr3Tu3DkFBQVp1apVmdq6spFLXsw3q3mYG9il09nZWT///LN+/fVXLV26VO+//76GDx+ujRs3qly5cpJkf0zS39//pscLAHmJO18AALs6depo165dCg0NVcWKFR1e1wpGly5d0pYtW+zv4+LidObMGVWrVs3eblxcXKY2K1asKCenG/+/og0bNmR6f6Wva83tWmOoVq2aDh065LD5w9X9XK1mzZpas2ZNtrspurq6Kj09/ZptVKtWzf75qCvWrVsnb29vlS5d+prnXsu9996rPXv2ZAo2Pj4+6tChgz766CPNmzdPX3/9tU6fPq06dero6NGjcnFxyXR9rmyLX7NmTS1fvjzbeVy6dEkbN260l506dUpxcXGqXr16jsbs6+uroKAghzYuXbqk6Ohoh3o2m00NGzbU2LFjtW3bNrm6ujp8Lmznzp0qUqSI7r777hz1CwBWIXwBAOz69eun06dPq1OnTtq8ebPi4+P1008/qUePHtcMEUWKFNGAAQO0ceNGRUdHq3v37rrvvvvsmzGMGjVKn376qcaOHatdu3YpNjZWX3zxhUaMGJGrca5bt06TJk3S77//rqlTp2r+/PkaOHDgNc+53hgiIiJUuXJldevWTdu3b9eaNWs0fPjwa7bZv39/JScnq2PHjtqyZYv27t2rzz77THFxcZIu78y4Y8cOxcXF6eTJk1mGtOeee06HDh3SgAEDtGfPHi1cuFCjR4/WkCFDchVMr3jggQd07tw57dq1y142efJkff7559qzZ49+//13zZ8/X4GBgfLz81NERITCw8PVpk0bLV26VAcOHNCvv/6q4cOH24P16NGj9fnnn2v06NGKjY3Vb7/9Zr9LWqlSJbVu3Vq9e/fW2rVrtX37dj399NMqVaqUWrduneNxDxw4UK+//rq+/fZb7dmzR88995zOnDljP75x40ZNmDBBW7Zs0cGDB7VgwQKdOHHCIXyvWbNGjRs3tvTOJADkBOELAGAXHBysdevWKT09XQ8//LBq1KihQYMGyc/P75pBwMPDQ6+88oo6d+6shg0bysvLS/PmzbMfj4yM1A8//KClS5eqfv36uu+++/TOO++obNmyuRrnCy+8oC1btujee+/Vv//9b02ePFmRkZHXPOd6Y3ByctI333yjv//+Ww0aNFCvXr2y3e7+iuLFi2vFihU6d+6cmjZtqrp16+qjjz6yP1rXu3dvValSRfXq1ZO/v7/WrVuXqY1SpUpp8eLF2rRpk2rVqqW+ffuqZ8+euQ6m/xzb448/rjlz5tjLvL29NWnSJNWrV0/169fXgQMHtHjxYjk5Oclms2nx4sVq0qSJevToocqVK6tjx476448/7J8NbNasmebPn6/vvvtOtWvX1oMPPqhNmzbZ2585c6bq1q2rRx99VOHh4TLGaPHixTf0mbgXXnhBXbp0Ubdu3RQeHi5vb2+H7ex9fHz0yy+/qFWrVqpcubJGjBiht99+Wy1btrTX+eKLL9S7d++buXwAcEvYzI08aA0AwFVmzZqlQYMGOdyduJVCQ0M1aNAgDRo0yJL+CrMdO3booYceUnx8vLy8vPJ7OJb48ccf9cILL2jHjh23/IusAeBGcecLAIDbVM2aNfXGG28oISEhv4dimZSUFM2cOZPgBaBA4n+ZAAC4jXXv3j2/h2CpJ554Ir+HAADZ4rFDAAAAALAAjx0CAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABb4fyjjOGXp74gIAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **TensorFlow Exercises** for Practice"
      ],
      "metadata": {
        "id": "6U6iPeDRU_uK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Building a model with the Sequential API\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "model_11 = Sequential([\n",
        "  layers.Input(shape=(1,), dtype=tf.string),\n",
        "  text_vectorizer,\n",
        "  embedding,\n",
        "  layers.GlobalAveragePooling1D(),\n",
        "  layers.Dense(1, activation=\"sigmoid\")\n",
        "], name=\"model_11_sequential\")"
      ],
      "metadata": {
        "id": "6ip8zj9TOD7R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile and fit the model\n",
        "model_11.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "model_11.fit(train_sentences, train_labels, epochs=5, validation_data=(val_sentences, val_labels))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kingG-ssUh6p",
        "outputId": "07c822b1-7af9-435a-c7c8-1865b3b32a73"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "215/215 [==============================] - 8s 32ms/step - loss: 0.4962 - accuracy: 0.8489 - val_loss: 0.4937 - val_accuracy: 0.7822\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 2s 7ms/step - loss: 0.2881 - accuracy: 0.9244 - val_loss: 0.4703 - val_accuracy: 0.7822\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.2090 - accuracy: 0.9412 - val_loss: 0.4879 - val_accuracy: 0.7795\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 2s 9ms/step - loss: 0.1661 - accuracy: 0.9555 - val_loss: 0.5163 - val_accuracy: 0.7598\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 1s 7ms/step - loss: 0.1380 - accuracy: 0.9620 - val_loss: 0.5487 - val_accuracy: 0.7559\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x7aa1e0194250>"
            ]
          },
          "metadata": {},
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Building another model with the Sequential API\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "model_21 = Sequential([\n",
        "  layers.Input(shape=(1,), dtype=tf.string),\n",
        "  text_vectorizer,\n",
        "  embedding,\n",
        "  layers.LSTM(64),\n",
        "  layers.Dense(1, activation=\"sigmoid\")\n",
        "], name=\"model_21_sequential\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FwrVDXhoTGj5",
        "outputId": "2dd15e8e-25dc-41db-dd0b-452cbc7d997a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile and fit the model\n",
        "model_21.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "model_21.fit(train_sentences, train_labels, epochs=5, validation_data=(val_sentences, val_labels))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jdYuIaGXU0x-",
        "outputId": "e7f053e3-35aa-4438-a65b-fa222ca29811"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "215/215 [==============================] - 15s 58ms/step - loss: 0.1144 - accuracy: 0.9654 - val_loss: 1.0839 - val_accuracy: 0.7415\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 6s 29ms/step - loss: 0.0503 - accuracy: 0.9790 - val_loss: 1.4567 - val_accuracy: 0.7178\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 7s 33ms/step - loss: 0.0460 - accuracy: 0.9800 - val_loss: 1.3286 - val_accuracy: 0.7375\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 5s 24ms/step - loss: 0.0404 - accuracy: 0.9822 - val_loss: 1.5000 - val_accuracy: 0.7113\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 7s 33ms/step - loss: 0.0412 - accuracy: 0.9813 - val_loss: 1.6233 - val_accuracy: 0.7231\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x7aa1e8e21630>"
            ]
          },
          "metadata": {},
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Building another model with the Sequential API\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "model_31 = Sequential([\n",
        "  layers.Input(shape=(1,), dtype=tf.string),\n",
        "  text_vectorizer,\n",
        "  embedding,\n",
        "  layers.Conv1D(filters=64, kernel_size=5, strides=1, activation=\"relu\", padding=\"valid\"),\n",
        "  layers.GlobalMaxPool1D(),\n",
        "  layers.Dense(1, activation=\"sigmoid\")\n",
        "], name=\"model_31_sequential\")"
      ],
      "metadata": {
        "id": "pmw-fDcJT2J1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile and fit the model\n",
        "model_31.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "model_31.fit(train_sentences, train_labels, epochs=5, validation_data=(val_sentences, val_labels))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sFhptX7bVCEd",
        "outputId": "7b622fe1-4759-45e0-f447-893825b96818"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "215/215 [==============================] - 9s 35ms/step - loss: 0.1002 - accuracy: 0.9666 - val_loss: 0.9979 - val_accuracy: 0.7349\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 2s 11ms/step - loss: 0.0638 - accuracy: 0.9764 - val_loss: 1.2199 - val_accuracy: 0.7257\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.0521 - accuracy: 0.9790 - val_loss: 1.2999 - val_accuracy: 0.7205\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 2s 8ms/step - loss: 0.0494 - accuracy: 0.9796 - val_loss: 1.3332 - val_accuracy: 0.7257\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 1s 7ms/step - loss: 0.0446 - accuracy: 0.9813 - val_loss: 1.4511 - val_accuracy: 0.7244\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x7aa1f81f7dc0>"
            ]
          },
          "metadata": {},
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Training baseline with 10% of the training data\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "# Create Tokenization and modelling pipeline\n",
        "model_41 = Pipeline([\n",
        "    (\"tfidf\", TfidfVectorizer()), # convert words to numbers using tfidf\n",
        "    (\"clf\", MultinomialNB()) # model the text\n",
        "])\n",
        "\n",
        "# Fit the pipeline to the training data\n",
        "model_41.fit(train_sentences_10_percent, train_labels_10_percent)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "id": "k6hFm2uAVMiE",
        "outputId": "7a2ab49e-1e34-44c7-a896-b0a659c67ea3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('tfidf', TfidfVectorizer()), ('clf', MultinomialNB())])"
            ],
            "text/html": [
              "<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 126
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "new_baseline_score = model_41.score(val_sentences,val_labels)\n",
        "print(f\"Our baseline model achieves an accuracy of: {new_baseline_score*100:.2f}%\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cOQFTXYTWKhq",
        "outputId": "a7e5145a-f8d6-4ba9-9bd8-cbefcd6a59fa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Our baseline model achieves an accuracy of: 73.75%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating a trainable sentence encoder layer\n",
        "sentence_encoder_layer_with_training = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder/4\",\n",
        "                                        input_shape=[], # shape of inputs coming to our model,\n",
        "                                        dtype=tf.string, # data type of inputs\n",
        "                                        trainable=True, # keep the pretrained weights (we'll create a feature extractor)\n",
        "                                        name=\"USE\")"
      ],
      "metadata": {
        "id": "L0bOA-8xWn74"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a model using the Sequential API\n",
        "model_51 = tf.keras.Sequential([\n",
        "  sentence_encoder_layer_with_training, # take in sentences and then encode them into an embedding\n",
        "  layers.Dense(64, activation=\"relu\"),\n",
        "  layers.Dense(1, activation=\"sigmoid\")],\n",
        "  name=\"model_51_USE\")\n",
        "\n",
        "# Compile the model\n",
        "model_51.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "qB1HB9MuYEB8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model\n",
        "model_51_history = model_51.fit(train_sentences,\n",
        "                                train_labels,\n",
        "                                epochs=5,\n",
        "                                validation_data=(val_sentences,val_labels))"
      ],
      "metadata": {
        "id": "7b1JFkbraRS6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e6a06a92-b04f-428d-aa48-c521af9e2cf2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "215/215 [==============================] - 32s 148ms/step - loss: 0.3245 - accuracy: 0.8682 - val_loss: 0.4262 - val_accuracy: 0.8136\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 26s 123ms/step - loss: 0.1141 - accuracy: 0.9628 - val_loss: 0.7102 - val_accuracy: 0.7546\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 25s 117ms/step - loss: 0.0534 - accuracy: 0.9823 - val_loss: 0.6209 - val_accuracy: 0.7940\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 22s 103ms/step - loss: 0.0349 - accuracy: 0.9876 - val_loss: 0.6530 - val_accuracy: 0.8058\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 23s 105ms/step - loss: 0.0264 - accuracy: 0.9901 - val_loss: 0.7139 - val_accuracy: 0.7913\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make some predictions with our Conv1D model\n",
        "model_51_pred_probs = model_51.predict(val_sentences)\n",
        "# Convert model 51 pred probs to labels\n",
        "model_51_preds = tf.squeeze(tf.round(model_51_pred_probs))\n",
        "# Evaluate model 51 predictions\n",
        "model_51_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_51_preds)\n",
        "model_51_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I4KdsDf7cQHH",
        "outputId": "85cf49f2-9464-40f6-a82e-768a09b4fa32"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 9ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 79.13385826771653,\n",
              " 'precision': 0.790794448403056,\n",
              " 'recall': 0.7913385826771654,\n",
              " 'f1': 0.7903304212685924}"
            ]
          },
          "metadata": {},
          "execution_count": 132
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Combine model results into a dataframe\n",
        "all_model_results = pd.DataFrame({\"0_baseline\": baseline_results,\n",
        "                                 \"1_simple_dense\": model_1_results,\n",
        "                                 \"2_lstm\": model_2_results,\n",
        "                                 \"3_gru\": model_3_results,\n",
        "                                 \"4_bidirectional\": model_4_results,\n",
        "                                 \"5_conv1d\": model_5_results,\n",
        "                                 \"6_tf_hub_use_encoder\": model_6_results,\n",
        "                                 \"7_tf_hub_use_encoder_10_percent\": model_7_results,\n",
        "                                 \"model_51_USE\": model_51_results})\n",
        "all_model_results = all_model_results.transpose()\n",
        "all_model_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331
        },
        "id": "om-YNg5Dapzs",
        "outputId": "715e66bf-7b73-46a0-a330-7a6950f29785"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                  accuracy  precision    recall        f1\n",
              "0_baseline                       79.790026   0.804912  0.797900  0.793292\n",
              "1_simple_dense                   78.346457   0.783490  0.783465  0.781572\n",
              "2_lstm                           73.622047   0.735678  0.736220  0.735887\n",
              "3_gru                            73.097113   0.730882  0.730971  0.730925\n",
              "4_bidirectional                  73.097113   0.733554  0.730971  0.731708\n",
              "5_conv1d                         71.916010   0.722271  0.719160  0.720003\n",
              "6_tf_hub_use_encoder             80.577428   0.805742  0.805774  0.804474\n",
              "7_tf_hub_use_encoder_10_percent  77.952756   0.779854  0.779528  0.779669\n",
              "model_51_USE                     79.133858   0.790794  0.791339  0.790330"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2fc1b609-6a73-47d7-90d4-1b3897393035\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0_baseline</th>\n",
              "      <td>79.790026</td>\n",
              "      <td>0.804912</td>\n",
              "      <td>0.797900</td>\n",
              "      <td>0.793292</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1_simple_dense</th>\n",
              "      <td>78.346457</td>\n",
              "      <td>0.783490</td>\n",
              "      <td>0.783465</td>\n",
              "      <td>0.781572</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2_lstm</th>\n",
              "      <td>73.622047</td>\n",
              "      <td>0.735678</td>\n",
              "      <td>0.736220</td>\n",
              "      <td>0.735887</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3_gru</th>\n",
              "      <td>73.097113</td>\n",
              "      <td>0.730882</td>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.730925</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4_bidirectional</th>\n",
              "      <td>73.097113</td>\n",
              "      <td>0.733554</td>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.731708</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5_conv1d</th>\n",
              "      <td>71.916010</td>\n",
              "      <td>0.722271</td>\n",
              "      <td>0.719160</td>\n",
              "      <td>0.720003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6_tf_hub_use_encoder</th>\n",
              "      <td>80.577428</td>\n",
              "      <td>0.805742</td>\n",
              "      <td>0.805774</td>\n",
              "      <td>0.804474</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7_tf_hub_use_encoder_10_percent</th>\n",
              "      <td>77.952756</td>\n",
              "      <td>0.779854</td>\n",
              "      <td>0.779528</td>\n",
              "      <td>0.779669</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>model_51_USE</th>\n",
              "      <td>79.133858</td>\n",
              "      <td>0.790794</td>\n",
              "      <td>0.791339</td>\n",
              "      <td>0.790330</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2fc1b609-6a73-47d7-90d4-1b3897393035')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2fc1b609-6a73-47d7-90d4-1b3897393035 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2fc1b609-6a73-47d7-90d4-1b3897393035');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-11f0e62a-6c5f-43a5-a9d6-36dddd2c2489\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-11f0e62a-6c5f-43a5-a9d6-36dddd2c2489')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-11f0e62a-6c5f-43a5-a9d6-36dddd2c2489 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_d2ca1fb7-e8d3-457c-af78-576992595dc5\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('all_model_results')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_d2ca1fb7-e8d3-457c-af78-576992595dc5 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('all_model_results');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "all_model_results",
              "summary": "{\n  \"name\": \"all_model_results\",\n  \"rows\": 9,\n  \"fields\": [\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.3959114273569506,\n        \"min\": 71.91601049868767,\n        \"max\": 80.5774278215223,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          78.34645669291339,\n          80.5774278215223,\n          79.79002624671917\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"precision\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.03412959359056996,\n        \"min\": 0.7222708272577039,\n        \"max\": 0.8057420069627774,\n        \"num_unique_values\": 9,\n        \"samples\": [\n          0.7798543396423501,\n          0.7834899124733871,\n          0.7222708272577039\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"recall\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.03395911427356953,\n        \"min\": 0.7191601049868767,\n        \"max\": 0.8057742782152231,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7834645669291339,\n          0.8057742782152231,\n          0.7979002624671916\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"f1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.032785639907219825,\n        \"min\": 0.7200034152753113,\n        \"max\": 0.8044743227851235,\n        \"num_unique_values\": 9,\n        \"samples\": [\n          0.7796693568429955,\n          0.7815717498014031,\n          0.7200034152753113\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Reduce the accuracy to the same scale as the other metrics\n",
        "all_model_results[\"accuracy\"] = all_model_results[\"accuracy\"]/100\n",
        "all_model_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331
        },
        "id": "0h-UEhcLd-Lf",
        "outputId": "9a60754c-2e89-4337-da40-4ae8faa6091e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                 accuracy  precision    recall        f1\n",
              "0_baseline                       0.797900   0.804912  0.797900  0.793292\n",
              "1_simple_dense                   0.783465   0.783490  0.783465  0.781572\n",
              "2_lstm                           0.736220   0.735678  0.736220  0.735887\n",
              "3_gru                            0.730971   0.730882  0.730971  0.730925\n",
              "4_bidirectional                  0.730971   0.733554  0.730971  0.731708\n",
              "5_conv1d                         0.719160   0.722271  0.719160  0.720003\n",
              "6_tf_hub_use_encoder             0.805774   0.805742  0.805774  0.804474\n",
              "7_tf_hub_use_encoder_10_percent  0.779528   0.779854  0.779528  0.779669\n",
              "model_51_USE                     0.791339   0.790794  0.791339  0.790330"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-29347745-25e5-414b-a3f7-e0991f50133e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0_baseline</th>\n",
              "      <td>0.797900</td>\n",
              "      <td>0.804912</td>\n",
              "      <td>0.797900</td>\n",
              "      <td>0.793292</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1_simple_dense</th>\n",
              "      <td>0.783465</td>\n",
              "      <td>0.783490</td>\n",
              "      <td>0.783465</td>\n",
              "      <td>0.781572</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2_lstm</th>\n",
              "      <td>0.736220</td>\n",
              "      <td>0.735678</td>\n",
              "      <td>0.736220</td>\n",
              "      <td>0.735887</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3_gru</th>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.730882</td>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.730925</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4_bidirectional</th>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.733554</td>\n",
              "      <td>0.730971</td>\n",
              "      <td>0.731708</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5_conv1d</th>\n",
              "      <td>0.719160</td>\n",
              "      <td>0.722271</td>\n",
              "      <td>0.719160</td>\n",
              "      <td>0.720003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6_tf_hub_use_encoder</th>\n",
              "      <td>0.805774</td>\n",
              "      <td>0.805742</td>\n",
              "      <td>0.805774</td>\n",
              "      <td>0.804474</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7_tf_hub_use_encoder_10_percent</th>\n",
              "      <td>0.779528</td>\n",
              "      <td>0.779854</td>\n",
              "      <td>0.779528</td>\n",
              "      <td>0.779669</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>model_51_USE</th>\n",
              "      <td>0.791339</td>\n",
              "      <td>0.790794</td>\n",
              "      <td>0.791339</td>\n",
              "      <td>0.790330</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-29347745-25e5-414b-a3f7-e0991f50133e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-29347745-25e5-414b-a3f7-e0991f50133e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-29347745-25e5-414b-a3f7-e0991f50133e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4d938d5b-32fc-497e-8bfa-576960c3e70c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4d938d5b-32fc-497e-8bfa-576960c3e70c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4d938d5b-32fc-497e-8bfa-576960c3e70c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_9a264a49-77a0-469e-9584-750c91a81e82\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('all_model_results')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_9a264a49-77a0-469e-9584-750c91a81e82 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('all_model_results');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "all_model_results",
              "summary": "{\n  \"name\": \"all_model_results\",\n  \"rows\": 9,\n  \"fields\": [\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0339591142735695,\n        \"min\": 0.7191601049868767,\n        \"max\": 0.8057742782152231,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7834645669291339,\n          0.8057742782152231,\n          0.7979002624671917\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"precision\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.03412959359056996,\n        \"min\": 0.7222708272577039,\n        \"max\": 0.8057420069627774,\n        \"num_unique_values\": 9,\n        \"samples\": [\n          0.7798543396423501,\n          0.7834899124733871,\n          0.7222708272577039\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"recall\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.03395911427356953,\n        \"min\": 0.7191601049868767,\n        \"max\": 0.8057742782152231,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7834645669291339,\n          0.8057742782152231,\n          0.7979002624671916\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"f1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.032785639907219825,\n        \"min\": 0.7200034152753113,\n        \"max\": 0.8044743227851235,\n        \"num_unique_values\": 9,\n        \"samples\": [\n          0.7796693568429955,\n          0.7815717498014031,\n          0.7200034152753113\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 135
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Plotting the results to comapre\n",
        "all_model_results.plot(kind=\"bar\", figsize=(10,7)).legend(bbox_to_anchor=(1.0, 1.0));"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 825
        },
        "id": "6O2MUiELdfsX",
        "outputId": "4250e325-0119-4245-c642-49ebbc38d6ba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7IAAAMoCAYAAADyfdzRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACApklEQVR4nOzdeVhUZePG8XtAARfAHdFQxJ3ElTRzV9K01yX9lVu5pWWKaaipuZdbmuuraW5pi7mUWb2aWaSZiKkoLrnjgpa4L4EJCvz+8GpqApRhOxz4fq5rrpc555mZm4kXueec8zyWxMTERAEAAAAAYBIORgcAAAAAAMAeFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYSh6jA6RGQkKCfv/9d7m6uspisRgdBwAAAIBBEhMT9ccff6hUqVJycOC4XG5liiL7+++/y8vLy+gYAAAAALKJ8+fP67HHHjM6BgxiiiLr6uoq6cEPq5ubm8FpAAAAABjl9u3b8vLysnYE5E6mKLJ/nU7s5uZGkQUAAADAJYe5HCeVAwAAAABMhSILAAAAADAViiwAAAAAwFRMcY0sAAAAAKRWQkKC4uLijI4BO+XNm1eOjo6pGkuRBQAAAJBjxMXF6cyZM0pISDA6CtKgUKFCKlmy5CMn86LIAgAAAMgREhMTdfHiRTk6OsrLy0sODlxJaRaJiYm6c+eOLl++LEny9PR86HiKLAAAAIAc4f79+7pz545KlSql/PnzGx0HdsqXL58k6fLlyypRosRDTzPmIwoAAAAAOUJ8fLwkycnJyeAkSKu/PoC4d+/eQ8dRZAEAAADkKI+6vhLZV2r/21FkAQAAAACmQpEFAAAAAJgKkz0BAAAAyNG8R27M0tc7O+3ZLH29zHDv3j3lzZvX6Bgp4ogsAAAAABhs8+bNatiwoQoVKqSiRYvqP//5jyIiIqz7L1y4oK5du6pIkSIqUKCA/P399csvv1j3f/PNN3riiSfk4uKiYsWK6bnnnrPus1gs2rBhg83rFSpUSCtWrJAknT17VhaLRWvWrFGTJk3k4uKiTz/9VNeuXVPXrl1VunRp5c+fX35+fvrss89snichIUHTp09XhQoV5OzsrDJlymjy5MmSpObNmyswMNBm/JUrV+Tk5KTg4OB0vV8UWQAAAAAwWExMjIKCgrR3714FBwfLwcFBzz33nBISEhQdHa0mTZrot99+09dff60DBw7ozTffVEJCgiRp48aNeu6559SmTRvt379fwcHBqlu3rt0ZRo4cqcGDB+vo0aNq1aqV7t69qzp16mjjxo06fPiwXnnlFb300kvavXu39TGjRo3StGnTNHbsWB05ckSrVq2Sh4eHJKlv375atWqVYmNjreM/+eQTlS5dWs2bN0/X+8WpxQAAAABgsE6dOtncX758uYoXL64jR45o586dunLlivbs2aMiRYpIkipUqGAdO3nyZHXp0kUTJ060bqtRo4bdGYYMGaKOHTvabBs2bJj160GDBum7777T2rVrVbduXf3xxx+aO3eu5s+fr549e0qSypcvr4YNG0qSOnbsqMDAQH311Vd64YUXJEkrVqxQr1690j2zNEdkAQAAAMBgJ0+eVNeuXeXj4yM3Nzd5e3tLkiIjIxUeHq5atWpZS+y/hYeHq0WLFunO4O/vb3M/Pj5e77zzjvz8/FSkSBEVLFhQ3333nSIjIyVJR48eVWxsbIqv7eLiopdeeknLly+XJO3bt0+HDx9Wr1690p2VI7IAAAAAYLC2bduqbNmyWrJkiUqVKqWEhARVq1ZNcXFxypcv30Mf+6j9FotFiYmJNtvu3buXZFyBAgVs7s+YMUNz587VnDlz5OfnpwIFCmjIkCGKi4tL1etKD04vrlmzpi5cuKAPP/xQzZs3V9myZR/5uEfhiCwAAAAAGOjatWs6fvy4xowZoxYtWqhq1aq6ceOGdX/16tUVHh6u69evJ/v46tWrP3TypOLFi+vixYvW+ydPntSdO3cemSskJETt27fXiy++qBo1asjHx0cnTpyw7q9YsaLy5cv30Nf28/OTv7+/lixZolWrVqlPnz6PfN3UoMgCAAAAgIEKFy6sokWLavHixTp16pR+/PFHBQUFWfd37dpVJUuWVIcOHRQSEqLTp0/riy++UGhoqCRp/Pjx+uyzzzR+/HgdPXpUhw4d0rvvvmt9fPPmzTV//nzt379fe/fuVf/+/VO1tE7FihX1/fffa+fOnTp69KheffVVXbp0ybrfxcVFI0aM0JtvvqmPPvpIERER2rVrl5YtW2bzPH379tW0adOUmJhoM5tyelBkAQAAAMBADg4OWr16tcLCwlStWjW98cYbmjFjhnW/k5OTtmzZohIlSqhNmzby8/PTtGnT5OjoKElq2rSp1q1bp6+//lo1a9ZU8+bNbWYWnjlzpry8vNSoUSN169ZNw4YNU/78+R+Za8yYMapdu7ZatWqlpk2bWsv0P40dO1ZDhw7VuHHjVLVqVXXu3FmXL1+2GdO1a1flyZNHXbt2lYuLSzreqb9ZEv99snQ2dPv2bbm7u+vWrVtyc3MzOg4AAAAAgzysG9y9e1dnzpxRuXLlMqwwIf3Onj2r8uXLa8+ePapdu/ZDx6b2vyGTPQEAAAAAMty9e/d07do1jRkzRk8++eQjS6w90nRq8YIFC+Tt7S0XFxfVq1fP5rB1cubMmaPKlSsrX7588vLy0htvvKG7d++mKTAAAAAAIPsLCQmRp6en9uzZo0WLFmXoc9t9RHbNmjUKCgrSokWLVK9ePc2ZM0etWrXS8ePHVaJEiSTjV61apZEjR2r58uV66qmndOLECesCuLNmzcqQbyJLTHC3Y+ytzMsBAAAynPfIjakee9alW6rG+ZUrk+rnXDv1fqrHVj12NNVjAcBITZs2TbLsT0ax+4jsrFmz1K9fP/Xu3Vu+vr5atGiR8ufPb13k9t927typBg0aqFu3bvL29lbLli3VtWvXRx7FBQAAAAAgOXYV2bi4OIWFhSkgIODvJ3BwUEBAgHXq53976qmnFBYWZi2up0+f1qZNm9SmTZsUXyc2Nla3b9+2uQEAAAAAINl5avHVq1cVHx8vDw8Pm+0eHh46duxYso/p1q2brl69qoYNGyoxMVH3799X//799dZbb6X4OlOnTtXEiRPtiQYAAAAAyCUyfR3Zbdu2acqUKXr//fe1b98+rV+/Xhs3btQ777yT4mNGjRqlW7duWW/nz5/P7JgAAAAAAJOw64hssWLF5OjoqEuXLtlsv3TpkkqWLJnsY8aOHauXXnpJffv2lST5+fkpJiZGr7zyikaPHi0Hh6Rd2tnZWc7OzvZESxP7JnZI/fP6rfRL9dhDPQ+l/okBAAAAAPYdkXVyclKdOnUUHBxs3ZaQkKDg4GDVr18/2cfcuXMnSVl1dHSUpEybwQoAAAAAkHPZfWpxUFCQlixZopUrV+ro0aN67bXXFBMTo969e0uSevTooVGjRlnHt23bVgsXLtTq1at15swZff/99xo7dqzatm1rLbQAAAAAgKyzbds2WSwW3bx5M0PHZhW715Ht3Lmzrly5onHjxikqKko1a9bU5s2brRNARUZG2hyBHTNmjCwWi8aMGaPffvtNxYsXV9u2bTV58uSM+y5M7GiVqqkax5pxAAAAADLKU089pYsXL8rd3T1Dx2YVu4usJAUGBiowMDDZfdu2bbN9gTx5NH78eI0fPz4tLwUAAADgH+yb56Vbqsf6lSuTqnFrp95P9XNmm4MxE7K4gE24lalPHxcXJycnp3Q9h5OTU4rzHKVnbFbJ9FmLAQAAAAApa9q0qfVgobu7u4oVK6axY8da5xTy9vbWO++8ox49esjNzU2vvPKKJGnHjh1q1KiR8uXLJy8vL73++uuKiYmxPm9sbKxGjBghLy8vOTs7q0KFClq2bJmkpKcLnzt3Tm3btlXhwoVVoEABPf7449q0aVOyYyXpiy++0OOPPy5nZ2d5e3tr5syZNt+Tt7e3pkyZoj59+sjV1VVlypTR4sWLM+w9o8gCAAAAgMFWrlypPHnyaPfu3Zo7d65mzZqlpUuXWve/9957qlGjhvbv36+xY8cqIiJCzzzzjDp16qSDBw9qzZo12rFjh82Zsz169NBnn32mefPm6ejRo/rggw9UsGDBZF9/4MCBio2N1fbt23Xo0CG9++67KY4NCwvTCy+8oC5duujQoUOaMGGCxo4dqxUrVtiMmzlzpvz9/bV//34NGDBAr732mo4fP57+N0tpPLUYAAAAAJBxvLy8NHv2bFksFlWuXFmHDh3S7Nmz1a9fP0lS8+bNNXToUOv4vn37qnv37hoyZIgkqWLFipo3b56aNGmihQsXKjIyUmvXrtX333+vgIAASZKPj0+Krx8ZGalOnTrJz8/vkWNnzZqlFi1aaOzYsZKkSpUq6ciRI5oxY4Z69eplHdemTRsNGDBAkjRixAjNnj1bW7duVeXKle1/g/6FI7IAAAAAYLAnn3xSFovFer9+/fo6efKk4uPjJUn+/v424w8cOKAVK1aoYMGC1lurVq2UkJCgM2fOKDw8XI6OjmrSpEmqXv/111/XpEmT1KBBA40fP14HDx5McezRo0fVoEEDm20NGjSwyStJ1atXt35tsVhUsmRJXb58OVV5HoUjsshwRk9AIEmHeh5K9VgAAAAguytQoIDN/ejoaL366qt6/fXXk4wtU6aMTp06Zdfz9+3bV61atdLGjRu1ZcsWTZ06VTNnztSgQYPSnDlv3rw29y0WixISEtL8fP9EkUWOxLJGAAAAMJNffvnF5v6uXbtUsWJFOTo6Jju+du3aOnLkiCpUqJDsfj8/PyUkJOinn36ynlr8KF5eXurfv7/69++vUaNGacmSJckW2apVqyokJMRmW0hIiCpVqpRi3oxGkQUAAECmMPosLVMuE4NcKzIyUkFBQXr11Ve1b98+/fe//00yE/A/jRgxQk8++aQCAwPVt29fFShQQEeOHNH333+v+fPny9vbWz179lSfPn00b9481ahRQ+fOndPly5f1wgsvJHm+IUOGqHXr1qpUqZJu3LihrVu3qmrV5A8ODR06VE888YTeeecdde7cWaGhoZo/f77ef//9DHs/HoUiCwAAAAAG69Gjh/7880/VrVtXjo6OGjx4sHWZneRUr15dP/30k0aPHq1GjRopMTFR5cuXV+fOna1jFi5cqLfeeksDBgzQtWvXVKZMGb311lvJPl98fLwGDhyoCxcuyM3NTc8884xmz56d7NjatWtr7dq1GjdunN555x15enrq7bfftpnoKbNRZAEAAADkbBNuGZ3gkfLmzas5c+Zo4cKFSfadPXs22cc88cQT2rJlS4rP6eLiolmzZmnWrFlJ9jVt2tS6Tq0k/fe//03xef49VpI6deqkTp06pfiY5DKHh4enON5ezFoMAAAAADAViiwAAAAAwFQ4tRgAAAAADLRt2zajI5gOR2QBAAAAAKZCkQUAAAAAmApFFgAAAABgKhRZAAAAAICpUGQBAAAAAKZCkQUAAAAAmApFFgAAAABymQkTJqhmzZrW+7169VKHDh0My2MviiwAAAAAwFTyGB0AAAAAADKT30q/LH29Qz0PpevxcXFxcnJyyqA0ORNHZAEAAADAQE2bNlVgYKCGDBmiYsWKqVWrVjp8+LBat26tggULysPDQy+99JKuXr1qfUxCQoKmT5+uChUqyNnZWWXKlNHkyZOt+0eMGKFKlSopf/788vHx0dixY3Xv3j0jvr1MQZEFAAAAAIOtXLlSTk5OCgkJ0bRp09S8eXPVqlVLe/fu1ebNm3Xp0iW98MIL1vGjRo3StGnTNHbsWB05ckSrVq2Sh4eHdb+rq6tWrFihI0eOaO7cuVqyZIlmz55txLeWKTi1GAAAAAAMVrFiRU2fPl2SNGnSJNWqVUtTpkyx7l++fLm8vLx04sQJeXp6au7cuZo/f7569uwpSSpfvrwaNmxoHT9mzBjr197e3ho2bJhWr16tN998M4u+o8xFkQUAAAAAg9WpU8f69YEDB7R161YVLFgwybiIiAjdvHlTsbGxatGiRYrPt2bNGs2bN08RERGKjo7W/fv35ebmlinZjUCRBQAAAACDFShQwPp1dHS02rZtq3fffTfJOE9PT50+ffqhzxUaGqru3btr4sSJatWqldzd3bV69WrNnDkzw3MbhSILAAAAANlI7dq19cUXX8jb21t58iStbBUrVlS+fPkUHBysvn37Jtm/c+dOlS1bVqNHj7ZuO3fuXKZmzmpM9gQAAAAA2cjAgQN1/fp1de3aVXv27FFERIS+++479e7dW/Hx8XJxcdGIESP05ptv6qOPPlJERIR27dqlZcuWSXpQdCMjI7V69WpFRERo3rx5+vLLLw3+rjIWRRYAAAAAspFSpUopJCRE8fHxatmypfz8/DRkyBAVKlRIDg4PKtzYsWM1dOhQjRs3TlWrVlXnzp11+fJlSVK7du30xhtvKDAwUDVr1tTOnTs1duxYI7+lDMepxQAAAABytEM9Dxkd4aG2bduWZFvFihW1fv36FB/j4OCg0aNH25w+/E/Tp0+3zoL8lyFDhli/njBhgiZMmGC9v2LFCnsiG44jsgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFSY7AkwCe+RG1M99qxLt1SP9StXJtVj1069n+qxVY8dTfVYAAAAwB4ckQUAAAAAmApHZAEAxprgbsfYW5mXAwAAmAZFFkCul9rTtjll2z6pf19T/5x+K/1SPTanvq8AAIBTiwEAAAAAJkORBQAAAAADJSYm6pVXXlGRIkVksVgUHh5udKRsjyILAAAAAAbavHmzVqxYof/973+6ePGibt++rbZt26pUqVKyWCzasGGD0RGzHa6RBQAAAJCjHa1SNUtfz965FyIiIuTp6amnnnpKkrR//37VqFFDffr0UceOHTMjoulRZAEAAADAIL169dLKlSslSRaLRWXLltXZs2fVunVrg5NlbxRZAAByqtQubcSyRgBgmLlz56p8+fJavHix9uzZI0dHR6MjmQJFFgAApJo9p+exrBEAPJq7u7tcXV3l6OiokiVLGh3HNCiyAACYSGrX55VSv0avXevzpnokAACZh1mLAQAAAACmQpEFAAAAAJgKpxYDAAAAQDYSHR2tU6dOWe+fOXNG4eHhKlKkiMqUKWNgsuyDIgsAAAAA2cjevXvVrFkz6/2goCBJUs+ePbVixQqDUmUvFFkAAAAAOVp2n0V9yJAhGjJkiPV+06ZNlZiYaFwgE+AaWQAAAACAqVBkAQAAAACmkqYiu2DBAnl7e8vFxUX16tXT7t27UxzbtGlTWSyWJLdnn302zaEBAAAAALmX3UV2zZo1CgoK0vjx47Vv3z7VqFFDrVq10uXLl5Mdv379el28eNF6O3z4sBwdHfX888+nOzwAAAAAIPexu8jOmjVL/fr1U+/eveXr66tFixYpf/78Wr58ebLjixQpopIlS1pv33//vfLnz0+RBQAAAACkiV1FNi4uTmFhYQoICPj7CRwcFBAQoNDQ0FQ9x7Jly9SlSxcVKFAgxTGxsbG6ffu2zQ0AAAAAUoMZf80rtf/t7CqyV69eVXx8vDw8PGy2e3h4KCoq6pGP3717tw4fPqy+ffs+dNzUqVPl7u5uvXl5edkTEwAAAEAu5OjoKOnBATiY0507dyRJefPmfei4LF1HdtmyZfLz81PdunUfOm7UqFHWRX8l6fbt25RZAAAAAA+VJ08e5c+fX1euXFHevHnl4MAiLWaRmJioO3fu6PLlyypUqJD1Q4mU2FVkixUrJkdHR126dMlm+6VLl1SyZMmHPjYmJkarV6/W22+//cjXcXZ2lrOzsz3RAAAAAORyFotFnp6eOnPmjM6dO2d0HKRBoUKFHtktJTuLrJOTk+rUqaPg4GB16NBBkpSQkKDg4GAFBgY+9LHr1q1TbGysXnzxRXteEgAAAABSzcnJSRUrVuT0YhPKmzfvI4/E/sXuU4uDgoLUs2dP+fv7q27dupozZ45iYmLUu3dvSVKPHj1UunRpTZ061eZxy5YtU4cOHVS0aFF7XxIAAAAAUs3BwUEuLi5Gx0AmsrvIdu7cWVeuXNG4ceMUFRWlmjVravPmzdYJoCIjI5Oci378+HHt2LFDW7ZsyZjUAAAAAIBcK02TPQUGBqZ4KvG2bduSbKtcuTJTYAMAAAAAMgTTeAEAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFNJU5FdsGCBvL295eLionr16mn37t0PHX/z5k0NHDhQnp6ecnZ2VqVKlbRp06Y0BQYAAAAA5G557H3AmjVrFBQUpEWLFqlevXqaM2eOWrVqpePHj6tEiRJJxsfFxenpp59WiRIl9Pnnn6t06dI6d+6cChUqlBH5AQAAAAC5jN1FdtasWerXr5969+4tSVq0aJE2btyo5cuXa+TIkUnGL1++XNevX9fOnTuVN29eSZK3t3f6UgMAAAAAci27Ti2Oi4tTWFiYAgIC/n4CBwcFBAQoNDQ02cd8/fXXql+/vgYOHCgPDw9Vq1ZNU6ZMUXx8fPqSAwAAAAByJbuOyF69elXx8fHy8PCw2e7h4aFjx44l+5jTp0/rxx9/VPfu3bVp0yadOnVKAwYM0L179zR+/PhkHxMbG6vY2Fjr/du3b9sTEwAAAACQg2X6rMUJCQkqUaKEFi9erDp16qhz584aPXq0Fi1alOJjpk6dKnd3d+vNy8srs2MCAAAAAEzCriJbrFgxOTo66tKlSzbbL126pJIlSyb7GE9PT1WqVEmOjo7WbVWrVlVUVJTi4uKSfcyoUaN069Yt6+38+fP2xAQAAAAA5GB2FVknJyfVqVNHwcHB1m0JCQkKDg5W/fr1k31MgwYNdOrUKSUkJFi3nThxQp6ennJyckr2Mc7OznJzc7O5AQAAAAAgpeHU4qCgIC1ZskQrV67U0aNH9dprrykmJsY6i3GPHj00atQo6/jXXntN169f1+DBg3XixAlt3LhRU6ZM0cCBAzPuuwAAAAAA5Bp2L7/TuXNnXblyRePGjVNUVJRq1qypzZs3WyeAioyMlIPD3/3Yy8tL3333nd544w1Vr15dpUuX1uDBgzVixIiM+y4AAAAAALmG3UVWkgIDAxUYGJjsvm3btiXZVr9+fe3atSstLwUAAAAAgI1Mn7UYAAAAAICMRJEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmkqYiu2DBAnl7e8vFxUX16tXT7t27Uxy7YsUKWSwWm5uLi0uaAwMAAAAAcje7i+yaNWsUFBSk8ePHa9++fapRo4ZatWqly5cvp/gYNzc3Xbx40Xo7d+5cukIDAAAAAHIvu4vsrFmz1K9fP/Xu3Vu+vr5atGiR8ufPr+XLl6f4GIvFopIlS1pvHh4e6QoNAAAAAMi97CqycXFxCgsLU0BAwN9P4OCggIAAhYaGpvi46OholS1bVl5eXmrfvr1+/fXXh75ObGysbt++bXMDAAAAAECys8hevXpV8fHxSY6oenh4KCoqKtnHVK5cWcuXL9dXX32lTz75RAkJCXrqqad04cKFFF9n6tSpcnd3t968vLzsiQkAAAAAyMEyfdbi+vXrq0ePHqpZs6aaNGmi9evXq3jx4vrggw9SfMyoUaN069Yt6+38+fOZHRMAAAAAYBJ57BlcrFgxOTo66tKlSzbbL126pJIlS6bqOfLmzatatWrp1KlTKY5xdnaWs7OzPdEAAAAAALmEXUdknZycVKdOHQUHB1u3JSQkKDg4WPXr10/Vc8THx+vQoUPy9PS0LykAAAAAALLziKwkBQUFqWfPnvL391fdunU1Z84cxcTEqHfv3pKkHj16qHTp0po6daok6e2339aTTz6pChUq6ObNm5oxY4bOnTunvn37Zux3AgAAAADIFewusp07d9aVK1c0btw4RUVFqWbNmtq8ebN1AqjIyEg5OPx9oPfGjRvq16+foqKiVLhwYdWpU0c7d+6Ur69vxn0XAAAAAIBcw+4iK0mBgYEKDAxMdt+2bdts7s+ePVuzZ89Oy8sAAAAAAJBEps9aDAAAAABARqLIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU0lTkV2wYIG8vb3l4uKievXqaffu3al63OrVq2WxWNShQ4e0vCwAAAAAAPYX2TVr1igoKEjjx4/Xvn37VKNGDbVq1UqXL19+6OPOnj2rYcOGqVGjRmkOCwAAAACA3UV21qxZ6tevn3r37i1fX18tWrRI+fPn1/Lly1N8THx8vLp3766JEyfKx8cnXYEBAAAAALmbXUU2Li5OYWFhCggI+PsJHBwUEBCg0NDQFB/39ttvq0SJEnr55ZdT9TqxsbG6ffu2zQ0AAAAAAMnOInv16lXFx8fLw8PDZruHh4eioqKSfcyOHTu0bNkyLVmyJNWvM3XqVLm7u1tvXl5e9sQEAAAAAORgmTpr8R9//KGXXnpJS5YsUbFixVL9uFGjRunWrVvW2/nz5zMxJQAAAADATPLYM7hYsWJydHTUpUuXbLZfunRJJUuWTDI+IiJCZ8+eVdu2ba3bEhISHrxwnjw6fvy4ypcvn+Rxzs7OcnZ2ticaAAAAACCXsOuIrJOTk+rUqaPg4GDrtoSEBAUHB6t+/fpJxlepUkWHDh1SeHi49dauXTs1a9ZM4eHhnDIMAAAAALCbXUdkJSkoKEg9e/aUv7+/6tatqzlz5igmJka9e/eWJPXo0UOlS5fW1KlT5eLiomrVqtk8vlChQpKUZDsAAAAAAKlhd5Ht3Lmzrly5onHjxikqKko1a9bU5s2brRNARUZGysEhUy+9BQAAAADkYnYXWUkKDAxUYGBgsvu2bdv20MeuWLEiLS8JAAAAAICkTJ61GAAAAACAjEaRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKmkqcguWLBA3t7ecnFxUb169bR79+4Ux65fv17+/v4qVKiQChQooJo1a+rjjz9Oc2AAAAAAQO5md5Fds2aNgoKCNH78eO3bt081atRQq1atdPny5WTHFylSRKNHj1ZoaKgOHjyo3r17q3fv3vruu+/SHR4AAAAAkPvYXWRnzZqlfv36qXfv3vL19dWiRYuUP39+LV++PNnxTZs21XPPPaeqVauqfPnyGjx4sKpXr64dO3akOzwAAAAAIPexq8jGxcUpLCxMAQEBfz+Bg4MCAgIUGhr6yMcnJiYqODhYx48fV+PGje1PCwAAAADI9fLYM/jq1auKj4+Xh4eHzXYPDw8dO3YsxcfdunVLpUuXVmxsrBwdHfX+++/r6aefTnF8bGysYmNjrfdv375tT0wAAAAAQA5mV5FNK1dXV4WHhys6OlrBwcEKCgqSj4+PmjZtmuz4qVOnauLEiVkRDQAAAABgMnYV2WLFisnR0VGXLl2y2X7p0iWVLFkyxcc5ODioQoUKkqSaNWvq6NGjmjp1aopFdtSoUQoKCrLev337try8vOyJCgAAAADIoey6RtbJyUl16tRRcHCwdVtCQoKCg4NVv379VD9PQkKCzanD/+bs7Cw3NzebGwAAAAAAUhpOLQ4KClLPnj3l7++vunXras6cOYqJiVHv3r0lST169FDp0qU1depUSQ9OE/b391f58uUVGxurTZs26eOPP9bChQsz9jsBAAAAAOQKdhfZzp0768qVKxo3bpyioqJUs2ZNbd682ToBVGRkpBwc/j7QGxMTowEDBujChQvKly+fqlSpok8++USdO3fOuO8CAAAAAJBrpGmyp8DAQAUGBia7b9u2bTb3J02apEmTJqXlZQAAAAAASMKua2QBAAAAADAaRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJhKmorsggUL5O3tLRcXF9WrV0+7d+9OceySJUvUqFEjFS5cWIULF1ZAQMBDxwMAAAAA8DB2F9k1a9YoKChI48eP1759+1SjRg21atVKly9fTnb8tm3b1LVrV23dulWhoaHy8vJSy5Yt9dtvv6U7PAAAAAAg97G7yM6aNUv9+vVT79695evrq0WLFil//vxavnx5suM//fRTDRgwQDVr1lSVKlW0dOlSJSQkKDg4ON3hAQAAAAC5j11FNi4uTmFhYQoICPj7CRwcFBAQoNDQ0FQ9x507d3Tv3j0VKVIkxTGxsbG6ffu2zQ0AAAAAAMnOInv16lXFx8fLw8PDZruHh4eioqJS9RwjRoxQqVKlbMrwv02dOlXu7u7Wm5eXlz0xAQAAAAA5WJbOWjxt2jStXr1aX375pVxcXFIcN2rUKN26dct6O3/+fBamBAAAAABkZ3nsGVysWDE5Ojrq0qVLNtsvXbqkkiVLPvSx7733nqZNm6YffvhB1atXf+hYZ2dnOTs72xMNAAAAAJBL2HVE1snJSXXq1LGZqOmviZvq16+f4uOmT5+ud955R5s3b5a/v3/a0wIAAAAAcj27jshKUlBQkHr27Cl/f3/VrVtXc+bMUUxMjHr37i1J6tGjh0qXLq2pU6dKkt59912NGzdOq1atkre3t/Va2oIFC6pgwYIZ+K0AAAAAAHIDu4ts586ddeXKFY0bN05RUVGqWbOmNm/ebJ0AKjIyUg4Ofx/oXbhwoeLi4vR///d/Ns8zfvx4TZgwIX3pAQAAAAC5jt1FVpICAwMVGBiY7L5t27bZ3D979mxaXgIAAAAAgGRl6azFAAAAAACkF0UWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYSpqK7IIFC+Tt7S0XFxfVq1dPu3fvTnHsr7/+qk6dOsnb21sWi0Vz5sxJa1YAAAAAAOwvsmvWrFFQUJDGjx+vffv2qUaNGmrVqpUuX76c7Pg7d+7Ix8dH06ZNU8mSJdMdGAAAAACQu9ldZGfNmqV+/fqpd+/e8vX11aJFi5Q/f34tX7482fFPPPGEZsyYoS5dusjZ2TndgQEAAAAAuZtdRTYuLk5hYWEKCAj4+wkcHBQQEKDQ0NAMCxUbG6vbt2/b3AAAAAAAkOwsslevXlV8fLw8PDxstnt4eCgqKirDQk2dOlXu7u7Wm5eXV4Y9NwAAAADA3LLlrMWjRo3SrVu3rLfz588bHQkAAAAAkE3ksWdwsWLF5OjoqEuXLtlsv3TpUoZO5OTs7Mz1tAAAAACAZNl1RNbJyUl16tRRcHCwdVtCQoKCg4NVv379DA8HAAAAAMC/2XVEVpKCgoLUs2dP+fv7q27dupozZ45iYmLUu3dvSVKPHj1UunRpTZ06VdKDCaKOHDli/fq3335TeHi4ChYsqAoVKmTgtwIAAAAAyA3sLrKdO3fWlStXNG7cOEVFRalmzZravHmzdQKoyMhIOTj8faD3999/V61ataz333vvPb333ntq0qSJtm3blv7vAAAAAACQq9hdZCUpMDBQgYGBye77dzn19vZWYmJiWl4GAAAAAIAksuWsxQAAAAAApIQiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFNJU5FdsGCBvL295eLionr16mn37t0PHb9u3TpVqVJFLi4u8vPz06ZNm9IUFgAAAAAAu4vsmjVrFBQUpPHjx2vfvn2qUaOGWrVqpcuXLyc7fufOneratatefvll7d+/Xx06dFCHDh10+PDhdIcHAAAAAOQ+dhfZWbNmqV+/furdu7d8fX21aNEi5c+fX8uXL092/Ny5c/XMM89o+PDhqlq1qt555x3Vrl1b8+fPT3d4AAAAAEDuY1eRjYuLU1hYmAICAv5+AgcHBQQEKDQ0NNnHhIaG2oyXpFatWqU4HgAAAACAh8ljz+CrV68qPj5eHh4eNts9PDx07NixZB8TFRWV7PioqKgUXyc2NlaxsbHW+7du3ZIk3b592564j5QQeyfVY29bElM9Nv7P+FSPjY5P3diM/t4zE+9r5jDT+yrlzPeW99U+vK+ZIzN+F/C+8r5mFqP/7eJ9zZnv61/PmZiY+u8NOY9dRTarTJ06VRMnTkyy3cvLy4A0D7jbNfpoqkfWTXUA+xKYBe9r5jD8fZVy5HvL+5o5eF8zT+q/K95Xe/C+Zo7M+F3A+5qz39c//vhD7jn0vxseza4iW6xYMTk6OurSpUs22y9duqSSJUsm+5iSJUvaNV6SRo0apaCgIOv9hIQEXb9+XUWLFpXFYrEncpa7ffu2vLy8dP78ebm5uRkdJ8fgfc0cvK+Zg/c1c/C+Zg7e18zB+5p5eG8zh5ne18TERP3xxx8qVaqU0VFgILuKrJOTk+rUqaPg4GB16NBB0oOSGRwcrMDAwGQfU79+fQUHB2vIkCHWbd9//73q16+f4us4OzvL2dnZZluhQoXsiWo4Nze3bP9LwIx4XzMH72vm4H3NHLyvmYP3NXPwvmYe3tvMYZb3lSOxsPvU4qCgIPXs2VP+/v6qW7eu5syZo5iYGPXu3VuS1KNHD5UuXVpTp06VJA0ePFhNmjTRzJkz9eyzz2r16tXau3evFi9enLHfCQAAAAAgV7C7yHbu3FlXrlzRuHHjFBUVpZo1a2rz5s3WCZ0iIyPl4PD3ZMhPPfWUVq1apTFjxuitt95SxYoVtWHDBlWrVi3jvgsAAAAAQK6RpsmeAgMDUzyVeNu2bUm2Pf/883r++efT8lKm4+zsrPHjxyc5NRrpw/uaOXhfMwfva+bgfc0cvK+Zg/c18/DeZg7eV5iNJZF5qwEAAAAAJuLw6CEAAAAAAGQfFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAgA9y7d099+vTRmTNnjI4CADkesxZnoLi4OJ05c0bly5dXnjxpWtkIyBJ79uzR1q1bdfnyZSUkJNjsmzVrlkGpAMD83N3dFR4ernLlyhkdBQByNNpWBrhz544GDRqklStXSpJOnDghHx8fDRo0SKVLl9bIkSMNTmhuERER+vDDDxUREaG5c+eqRIkS+vbbb1WmTBk9/vjjRscznSlTpmjMmDGqXLmyPDw8ZLFYrPv++TVglI4dO6Z67Pr16zMxSc7y9ddfp3psu3btMjFJztahQwdt2LBBb7zxhtFRchRHR0ddvHhRJUqUsNl+7do1lShRQvHx8QYlM6fp06dr0KBBypcvnyQpJCRE/v7+1jVk//jjD40YMULvv/++kTGBh+KIbAYYPHiwQkJCNGfOHD3zzDM6ePCgfHx89NVXX2nChAnav3+/0RFN66efflLr1q3VoEEDbd++XUePHpWPj4+mTZumvXv36vPPPzc6oul4eHjo3XffVa9evYyOkqOUK1fuoR8EnD59OgvTmFvv3r1TPfbDDz/MxCQ5i4OD7dVEFotF//wT4J8/v5SCtJs0aZJmzpypFi1aqE6dOipQoIDN/tdff92gZObm4OCgqKioJEX2999/V/ny5fXnn38alMyc/v3BgJubm8LDw+Xj4yNJunTpkkqVKsXvAmRrHJHNABs2bNCaNWv05JNP2vwh8PjjjysiIsLAZOY3cuRITZo0SUFBQXJ1dbVub968uebPn29gMvNycHBQgwYNjI6R4wwZMsTm/r1797R//35t3rxZw4cPNyaUSVFOM8c/LyP44YcfNGLECE2ZMkX169eXJIWGhmrMmDGaMmWKURFzhGXLlqlQoUIKCwtTWFiYzT6LxUKRtdO8efMkPXjvli5dqoIFC1r3xcfHa/v27apSpYpR8Uzr38exOK4FM6LIZoArV64k+YRQkmJiYjhVM50OHTqkVatWJdleokQJXb161YBE5vfGG29owYIFmjNnjtFRcpTBgwcnu33BggXau3dvFqcBHm7IkCFatGiRGjZsaN3WqlUr5c+fX6+88oqOHj1qYDpzY6KnjDV79mxJD4rWokWL5OjoaN3n5OQkb29vLVq0yKh4AAxEkc0A/v7+2rhxowYNGiTp79Ozli5dav2kG2lTqFAhXbx4McmkGfv371fp0qUNSmVuw4YN07PPPqvy5cvL19dXefPmtdnPNYcZq3Xr1ho1ahRHGdPh888/19q1axUZGam4uDibffv27TMolblFRESoUKFCSba7u7vr7NmzWZ4nJ2ICyIzx1wcDzZo10/r161W4cGGDEwHILvjNmgGmTJmi1q1b68iRI7p//77mzp2rI0eOaOfOnfrpp5+MjmdqXbp00YgRI7Ru3TpZLBYlJCQoJCREw4YNU48ePYyOZ0qvv/66tm7dqmbNmqlo0aKcNZDJPv/8cxUpUsToGKY1b948jR49Wr169dJXX32l3r17KyIiQnv27NHAgQONjmdaTzzxhIKCgvTxxx/Lw8ND0oNr4oYPH666desanM7cmAAyc2zdutXoCDnOP0/Vvn//vlasWKFixYpJejDZE5DdMdlTBomIiNC0adN04MABRUdHq3bt2hoxYoT8/PyMjmZqcXFxGjhwoFasWKH4+HjlyZNH8fHx6tatm1asWGFzihFSx9XVVatXr9azzz5rdJQcpVatWjYfCiQmJioqKkpXrlzR+++/r1deecXAdOZVpUoVjR8/Xl27dpWrq6sOHDggHx8fjRs3TtevX+da+TQ6deqUnnvuOZ04cUJeXl6SpPPnz6tixYrasGGDKlSoYHBC82ICyMwRHx+vFStWKDg4ONml43788UeDkpmTt7d3qj7I5lR5ZGcUWZjC+fPndejQIUVHR6tWrVqqWLGi0ZFMq2zZsvruu++YHCODTZw40ea+g4ODihcvrqZNm/Jep0P+/Pl19OhRlS1bViVKlND333+vGjVq6OTJk3ryySd17do1oyOaVmJior7//nsdO3ZMklS1alUFBARwlkY6lS1b1joB5D8/fDl16pRq166t27dvGx3RlAIDA7VixQo9++yz8vT0TPJz+te1tAByD04tziAJCQk6depUsp8SNm7c2KBUOYeXl5e8vLwUHx+vQ4cO6caNG1wnk0YTJkzQ+PHj9eGHHyp//vxGx8kR7t+/r3LlyqlVq1bW0zSRMUqWLKnr16+rbNmyKlOmjHbt2qUaNWrozJkzzLKZThaLRS1btlTLli2NjpKjMAFk5li9erXWrl2rNm3aGB0FQDbh8OgheJRdu3apQoUKqlq1qho3bqymTZtab82aNTM6nqkNGTJEy5Ytk/TgtKImTZqodu3a8vLy0rZt24wNZ1Lz5s3Tt99+Kw8PD/n5+al27do2N9gvT5486t+/v+7evWt0lBynefPm+vrrryU9WF/2jTfe0NNPP63OnTvrueeeMzhdznPp0iW9/fbbRscwtb8mgPwLE0BmDCcnJ055z0ChoaH63//+Z7Pto48+Urly5VSiRAm98sorio2NNSgdkDockc0A/fv3t/7DldzpLki7zz//XC+++KIk6ZtvvtHp06d17Ngxffzxxxo9erRCQkIMTmg+7du352c0E9StW1f79+9X2bJljY6SoyxevNh6lsvAgQNVtGhR7dy5U+3atdOrr75qcLqcJyoqShMnTtS4ceOMjmJaTACZOYYOHaq5c+dq/vz5/BuWAd5++201bdpU//nPfyQ9WO7w5ZdfVq9evVS1alXNmDFDpUqV0oQJE4wNCjwE18hmgAIFCujAgQN8UpgJXFxcdOrUKT322GN65ZVXlD9/fs2ZM0dnzpxRjRo1uNYI2cbatWs1atQovfHGG6pTp44KFChgs7969eoGJQP+dvDgwYfuP3bsmLp27ar4+PgsSpQzMQFkxnvuuee0detWFSlSRI8//jhLx6WTp6envvnmG/n7+0uSRo8erZ9++kk7duyQJK1bt07jx4/XkSNHjIwJPBRHZDNAvXr1dOrUKYpsJvDw8NCRI0fk6empzZs3a+HChZIeLG/AjMVp4+Pjoz179qho0aI222/evKnatWvr9OnTBiUzty5dukh6sLzRXywWixITE2WxWCgG6XDz5k3t3r072TkIWIbLPjVr1rT+XP7bP39ekT7ly5fXkiVLjI6RoxQqVIjLCTLQjRs3bOZ0+Omnn9S6dWvr/SeeeELnz583IhqQahTZDDBo0CANHTpUUVFR8vPzS/IpIUdi0q5379564YUXrKdsBwQESJJ++eUXZoJNo7NnzyZbqmJjY3XhwgUDEuUMLFGQOb755ht1795d0dHRcnNzsylZFouFImunIkWKaPr06WrRokWy+3/99Ve1bds2i1OZnz1nB7m5uWVikpzrww8/NDpCjuLh4aEzZ87Iy8tLcXFx2rdvn83s+3/88UeSv2eB7IYimwE6deokSerTp491G0diMsaECRNUrVo1nT9/Xs8//7ycnZ0lSY6Ojiwqb6e/JsyRpO+++07u7u7W+/Hx8QoODla5cuWMiJYjcG1s5hg6dKj69OmjKVOmMMt2BqhTp45+//33FH9eb968yWzQaVCoUKFUH8nmb4K0u3//vrZt26aIiAh169ZNrq6u+v333+Xm5qaCBQsaHc9U2rRpo5EjR+rdd9/Vhg0blD9/fjVq1Mi6/+DBgypfvryBCYFH4xrZDHDu3LmH7ucPXGQHDg4PJilP7rTCvHnzytvbWzNnzrRO/AD7/PODgn+yWCxycXFRhQoV+KAgDQoUKKBDhw7Jx8fH6Cg5wpdffqmYmBjrJHr/duPGDX399dfq2bNnFiczt39O4nT27FmNHDlSvXr1ss5SHBoaqpUrV2rq1Km8t2l07tw5PfPMM4qMjFRsbKxOnDghHx8fDR48WLGxsVq0aJHREU3l6tWr6tixo3bs2KGCBQtq5cqVNqdut2jRQk8++aQmT55sYErg4SiyyPaCg4MVHByc7PVxy5cvNyiVeZUrV0579uxRsWLFjI6Sozg4OCT7IcE/z85o2LChNmzYwBrIdujYsaO6dOmiF154wegoQKq0aNFCffv2VdeuXW22r1q1SosXL2bpuDTq0KGDXF1dtWzZMhUtWlQHDhyQj4+Ptm3bpn79+unkyZNGRzSlW7duqWDBgknmHbl+/boKFiwoJycng5IBj8apxWn09ddfq3Xr1sqbN2+KR2L+0q5duyxKlfNMnDhRb7/9tvz9/VnaKIMkdy3nzZs3VahQoawPk4N8//33Gj16tCZPnqy6detKknbv3q2xY8dqzJgxcnd316uvvqphw4ZZ10bGoz377LMaPny4jhw5kuwcBPx+TZsdO3aoYcOGRsfIkUJDQ5M9Oujv76++ffsakChn+Pnnn7Vz584kxcrb21u//fabQanM75+XGf1TkSJFsjgJYD+OyKaRg4ODoqKiVKJECespm8nhGtn08fT01PTp0/XSSy8ZHSXHePfdd+Xt7a3OnTtLkp5//nl98cUX8vT01KZNm1SjRg2DE5pTtWrVtHjxYj311FM220NCQvTKK6/o119/1Q8//KA+ffooMjLSoJTmw+/XzOHk5KTSpUura9euevHFF+Xr62t0pByjcuXKat++vaZPn26z/c0339RXX32l48ePG5TM3AoXLqyQkBD5+vrK1dXVekR2x44d6tSpky5dumR0RFOpVatWsgcH3N3dValSJQ0ZMkRVq1Y1IBmQehyRTaN/nuL679NdkXHi4uKSFAOkz6JFi/Tpp59KenAU8YcfftDmzZu1du1aDR8+XFu2bDE4oTlFREQkOxupm5ubdUmjihUr6urVq1kdzdT4/Zo5fv/9d61evVqfffaZpk2bpurVq6t79+7q2rWrHnvsMaPjmdrs2bPVqVMnffvtt6pXr56kB2dnnDx5Ul988YXB6cyrZcuWmjNnjhYvXizpwQdZ0dHRGj9+vNq0aWNwOvPp0KFDsttv3rypffv2qWbNmvrxxx/VoEGDrA0G2IEjssjWRowYoYIFC2rs2LFGR8kx8uXLpxMnTsjLy0uDBw/W3bt39cEHH+jEiROqV6+ebty4YXREU2rYsKFcXV310UcfqXjx4pKkK1euqEePHoqJidH27dv1ww8/aODAgRyRQbZy5swZrVq1Sp999pmOHTumxo0b68cffzQ6lqlduHBB77//vo4dOyZJqlq1qvr37y8vLy+Dk5nXhQsX1KpVKyUmJurkyZPy9/fXyZMnVaxYMW3fvl0lSpQwOmKOMnr0aO3atUvBwcFGRwFSRJFNo3nz5qV67Ouvv56JSXK2wYMH66OPPlL16tVVvXr1JNfHzZo1y6Bk5lWqVCl9/vnneuqpp1S5cmVNmjRJzz//vI4fP64nnnjCrvUQ8bfjx4+rffv21nX5JOn8+fPy8fHRV199pUqVKmnDhg36448/OFXeTj/99JPee+89HT16VJLk6+ur4cOH2ywVgfSJj4/Xt99+q7Fjx+rgwYOcso1s6f79+1qzZo0OHDig6Oho1a5dW927d1e+fPmMjpbj/Prrr2rWrJkuX75sdBQgRRTZNErtMhoWi8V6WiHs16xZsxT3WSwWjhqkQWBgoP73v/+pYsWK2r9/v86ePauCBQtq9erVmj59uvbt22d0RNNKSEjQli1bdOLECUkPrpV7+umnH3qdJx7uk08+Ue/evdWxY0frKW4hISH68ssvtWLFCnXr1s3ghOYWEhKiTz/9VJ9//rnu3r2r9u3bq3v37nrmmWeMjmZqN2/e1LJly6wfvjz++OPq06dPihPrANnNsWPH1LBhQy6HQbZGkQVymXv37mnu3Lk6f/68evXqpVq1akl6cF2Xq6srs2pmMj8/P23atIlTDFOpatWqeuWVV/TGG2/YbJ81a5aWLFliLQqwz6hRo7R69Wr9/vvvevrpp9W9e3e1b99e+fPnNzqa6e3du1etWrVSvnz5rDOY79mzR3/++ae2bNmi2rVrG5zQnKZOnSoPDw/16dPHZvvy5ct15coVjRgxwqBkOdOUKVO0efNmbd++3egoQIooshkoLi5OZ86cUfny5ZUnD/NoZaRTp04pIiJCjRs3Vr58+azrcgJm88/ZNvFozs7O+vXXX1WhQgWb7adOnVK1atV09+5dg5KZW4MGDdS9e3e98MILrCmdwRo1aqQKFSpoyZIl1r8F7t+/r759++r06dMUgzTy9vbWqlWrkkwA+csvv6hLly7JLi2HlKV0idytW7cUFhamjRs36ttvv1VAQEAWJwNSj7aVAe7cuaNBgwZp5cqVkqQTJ07Ix8dHgwYNUunSpTVy5EiDE5rXtWvX9MILL2jr1q2yWCw6efKkfHx89PLLL6tw4cKaOXOm0RFN4VFrHf8T63IiO/Hy8lJwcHCSIvvDDz9wVDsdQkJCjI6QY+3du9emxEpSnjx59Oabb8rf39/AZOYWFRUlT0/PJNuLFy+uixcvGpDI3GbPnp3sdjc3N1WuXFnbt29X/fr1szgVYB+KbAYYNWqUDhw4oG3bttlcVxQQEKAJEyZQZNPhjTfeUN68eRUZGWmznlnnzp0VFBREkU2llKbZ/zfW5UR2M3ToUL3++usKDw+3HokJCQnRihUrNHfuXIPTmdvJkye1detWXb58OckyR+PGjTMolfm5ubkpMjJSVapUsdl+/vx5ubq6GpTK/Ly8vBQSEpJkjpKQkBCVKlXKoFTmxRFs5AQU2QywYcMGrVmzRk8++aTN6a6PP/64IiIiDExmflu2bNF3332XZF3DihUr6ty5cwalMh/W4oRZvfbaaypZsqRmzpyptWvXSnpw3eyaNWvUvn17g9OZ15IlS/Taa6+pWLFiKlmypM2/XRaLhSKbDp07d9bLL7+s9957z+bDl+HDh6tr164GpzOvfv36aciQIbp3756aN28uSQoODtabb76poUOHGpwu53Nzc1N4eDiXxSBbochmgCtXriS7fllMTAzXcaZTTExMspOPXL9+Xc7OzgYkyj2YlAjZxXPPPafnnnvO6Bg5yqRJkzR58mQmyMkE7733niwWi3r06KH79+9LkvLmzavXXntN06ZNMzideQ0fPlzXrl3TgAEDFBcXJ0lycXHRiBEjNGrUKIPT5XxMqYPsiDUhMoC/v782btxovf9XeV26dCnXF6RTo0aN9NFHH1nvWywWJSQkaPr06Q9dmgfpd/bsWd27d8/oGAAywY0bN/T8888bHSNHcnJy0ty5c3Xjxg2Fh4crPDxc169f1+zZs/kANo3i4+P1888/a+TIkbpy5Yp27dqlAwcO6Pr165w9AORiHJHNAFOmTFHr1q115MgR3b9/X3PnztWRI0e0c+dO/fTTT0bHM7Xp06erRYsW2rt3r+Li4vTmm2/q119/1fXr15msBNnG0aNHtWvXLtWvX19VqlTRsWPHNHfuXMXGxurFF1+0ngYnSR988IE8PDwMTJv9FSlSRCdOnFCxYsVUuHDhh57Zcv369SxMlnM8//zz2rJli/r37290lBzn1q1bio+PV5EiReTn52fdfv36deXJk0dubm4GpjMnR0dHtWzZUkePHlW5cuX0xBNPGB0JQDZAkc0ADRs2VHh4uKZNmyY/Pz/rOnGhoaE2/4jBftWqVdOJEyc0f/58ubq6Kjo6Wh07dtTAgQOTnb0QyGqbN29W+/btVbBgQd25c0dffvmlevTooRo1aighIUEtW7bUli1brGW2W7duBifO/v5a0/ivr7lEI+NVqFBBY8eO1a5du+Tn56e8efPa7H/99dcNSmZ+Xbp0Udu2bTVgwACb7WvXrtXXX3+tTZs2GZTM3KpVq6bTp08nmewJQO7FOrIAksV6p6nz1FNPqXnz5po0aZJWr16tAQMG6LXXXtPkyZMlPZjVPCwsTFu2bDE4KfC3h5UBi8Wi06dPZ2GanKVIkSIKCQmxmWlfko4dO6YGDRro2rVrBiUzt82bN2vUqFF65513VKdOHRUoUMBmP0e6MxeTPSE7oshmgH379ilv3rzWo69fffWVPvzwQ/n6+mrChAlycnIyOKG5HDx4MNVjq1evnolJcjeKbOq4u7srLCxMFSpUUEJCgpydnbV7927VqlVLknT48GEFBAQoKirK4KTm5OjoqIsXLyaZUO/atWsqUaIEy0Uh2ylQoID1SPc/HTp0SPXq1dOdO3cMSmZuDg5/T+vyz7M0EhMTWTouC/A3AbIjTi3OAK+++qpGjhwpPz8/nT59Wp07d1bHjh21bt063blzR3PmzDE6oqnUrFlTFovF+o/TX/76zOWf2/iHC9nBXz+TDg4OcnFxkbu7u3Wfq6urbt26ZVQ000vps9bY2Fg+JMwgyf1uRdrVrVtXixcv1n//+1+b7YsWLVKdOnUMSmV+W7duNTpCrvbtt9+qdOnSRscAbFBkM8CJEydUs2ZNSdK6devUpEkTrVq1SiEhIerSpQtF1k7/XKR7//79GjZsmIYPH26dATo0NFQzZ87U9OnTjYqYKzApUep4e3vr5MmTKl++vKQHP59lypSx7o+MjOR67jSYN2+epAflaunSpSpYsKB1X3x8vLZv364qVaoYFS9H+OijjzRjxgydPHlSklSpUiUNHz5cL730ksHJzG3SpEkKCAjQgQMH1KJFC0kP1jvds2cPlxikQ5MmTYyOkKucP39e48eP1/LlyyU9mA8GyG4oshkgMTFRCQkJkqQffvhB//nPfyRJXl5eunr1qpHRTKls2bLWr59//nnNmzdPbdq0sW6rXr26vLy8NHbsWHXo0MGAhOb1559/KiwsTEWKFJGvr6/Nvrt372rt2rXq0aOHJCYlSq3XXnvN5syAatWq2ez/9ttvbWYtRurMnj1b0oPfr4sWLZKjo6N1n5OTk7y9vbVo0SKj4pnerFmzNHbsWAUGBqpBgwaSpB07dqh///66evWq3njjDYMTmleDBg0UGhqqGTNmaO3atcqXL5+qV6+uZcuWqWLFikbHM7Wff/5ZH3zwgU6fPq1169apdOnS+vjjj1WuXDmKVga7fv26Vq5caS2yQHbENbIZoHnz5vLy8lJAQIBefvllHTlyRBUqVNBPP/2knj176uzZs0ZHNK18+fJp3759SSbNOHr0qGrXrq0///zToGTmc+LECbVs2VKRkZGyWCxq2LChVq9ebT1aeOnSJZUqVYrTtZGtNGvWTOvXr1fhwoWNjpKjlCtXThMnTrR+cPWXlStXasKECTZnxgDZwRdffKGXXnpJ3bt318cff6wjR47Ix8dH8+fP16ZNm5gN2k5ff/31Q/efPn1aQ4cO5W8CZGsU2Qxw8OBBde/eXZGRkQoKCtL48eMlSYMGDdK1a9e0atUqgxOaV+3atVWtWjUtXbrUej1cXFyc+vbtq8OHD2vfvn0GJzSP5557Tvfu3dOKFSt08+ZNDRkyREeOHNG2bdtUpkwZiiyQi7i4uOjw4cOqUKGCzfaTJ0/Kz89Pd+/eNShZzpCQkKBTp07p8uXL1jO2/tK4cWODUplbrVq19MYbb6hHjx42Ew/t379frVu3ZkI9Ozk4OFjnI0kJk2ghu6PIZqK7d+/K0dExyfp8SL3du3erbdu2SkxMtM5QfPDgQVksFn3zzTeqW7euwQnNw8PDQz/88IN1Js3ExEQNGDBAmzZt0tatW1WgQAGKLLKdTp06qW7duhoxYoTN9unTp2vPnj1at26dQcnMrVq1aurWrZveeustm+2TJk3SmjVrdOjQIYOSmd+uXbvUrVs3nTt3LklJoBikXf78+XXkyBF5e3vbFNnTp0/L19eXD1/sVLp0ab3//vtq3759svvDw8NVp04dfl6RrXGNbCZycXExOoLp1a1bV6dPn9ann36qY8eOSZI6d+6sbt26JVlDDg/3559/Kk+ev/8vb7FYtHDhQgUGBlonKAOym+3bt2vChAlJtrdu3VozZ87M+kA5xMSJE9W5c2dt377deo1sSEiIgoODtXbtWoPTmVv//v3l7++vjRs3ytPTk9mgM0jJkiV16tQpeXt722zfsWMHS8KkQZ06dRQWFpZikX3U0VogO6DIZoD4+HjNnj1ba9euVWRkpOLi4mz2X79+3aBkOUOBAgX0yiuvPHTMs88+q6VLlzI77ENUqVJFe/fuTXK98fz58yVJ7dq1MyIW8FDR0dHJLrOTN29e3b5924BEOUOnTp30yy+/aPbs2dqwYYMkqWrVqjZrICNtTp48qc8//zzJadtIn379+mnw4MFavny5LBaLfv/9d4WGhmrYsGEaO3as0fFMZ/jw4YqJiUlxf4UKFVjyCNkeRTYDTJw4UUuXLtXQoUM1ZswYjR49WmfPntWGDRs0btw4o+PlCtu3b2fip0d47rnn9NlnnyW7tMb8+fOVkJDALLDIdvz8/LRmzZokv0tXr16dZOZt2KdOnTr65JNPjI6R49SrV0+nTp2iyGawkSNHKiEhQS1atNCdO3fUuHFjOTs7a9iwYRo0aJDR8UynUaNGD91foEABljxCtsc1shmgfPnymjdvnp599lm5uroqPDzcum3Xrl2cspkF/nm9DICc45tvvlHHjh3VrVs36zJGwcHB+uyzz7Ru3TqW4EqjTZs2ydHRUa1atbLZ/t133ykhIUGtW7c2KJn5ffnllxozZoyGDx8uPz+/JPNk/DXfA9ImLi5Op06dUnR0tHx9fW3WmAaQu1BkM0CBAgV09OhRlSlTRp6entq4caNq166t06dPq1atWrp165bREXM8iiyQc23cuFFTpkxReHi4dU3O8ePHc7QgHapXr65p06bZrNEtSZs3b9aIESN04MABg5KZn4ODQ5Jtf11vyGRPGeP8+fOSJC8vL4OTmNfly5dVokQJ6/3w8HDNnj1bp06dkqenpwIDA9W0aVPjAgKpkPS3Lez22GOP6eLFi5IeHJ3dsmWLJGnPnj1ydnY2MhoAmN6zzz6rkJAQxcTE6OrVq/rxxx8psel08uTJZE/NrlKlik6dOmVAopzjzJkzSW6nT5+2/i/S5v79+xo7dqzc3d3l7e0tb29vubu7a8yYMbp3757R8UzH09NTly9fliTt3LlTdevW1blz59SgQQPdvn1bTz/9tLZv325wSuDhuEY2Azz33HMKDg5WvXr1NGjQIL344otatmyZIiMj9cYbbxgdDwBM7ebNm/r88891+vRpDRs2TEWKFNG+ffvk4eGh0qVLGx3PlNzd3XX69OkkM8CeOnWKGeHTqWzZskZHyJEGDRqk9evXa/r06apfv74kKTQ0VBMmTNC1a9e0cOFCgxOayz9PyJwwYYJeeuklLVu2zLptyJAhmjhxooKDg42IB6QKpxZngtDQUIWGhqpixYpq27at0XFyBU4tBnKmgwcPKiAgQO7u7jp79qyOHz8uHx8fjRkzRpGRkfroo4+MjmhKr776qkJDQ/Xll1+qfPnykh6U2E6dOumJJ57Q0qVLDU5obh9//LEWLVqkM2fOKDQ0VGXLltWcOXNUrly5FJc7wcO5u7tr9erVSa7f3rRpk7p27cplXHZycHBQVFSUSpQooVKlSmn9+vV68sknrft//fVXNW3aVFeuXDEwJfBwnFqcCerXr6+goCBKbBZ66623VKRIEaNjAMhgQUFB6tWrl06ePGmzNnebNm047S0dpk+frgIFCqhKlSoqV66cypUrp6pVq6po0aJ67733jI5nagsXLlRQUJDatGmjmzdvWq+JLVSokObMmWNsOBNzdnZOcgaBJJUrVy7ZJbrwaH/88Ydu374tFxeXJJfCubi46M6dOwYlA1KHIptBjh8/rsDAQLVo0UItWrRQYGCgjh8/bnSsHOHjjz9WgwYNVKpUKZ07d06SNGfOHH311VfWMaNGjVKhQoUMSgggs+zZs0evvvpqku2lS5dWVFSUAYlyBnd3d+3cuVMbN27UgAEDNHToUAUHB+vHH3/kd2k6/fe//9WSJUs0evRoOTo6Wrf7+/vr0KFDBiYzt8DAQL3zzjuKjY21bouNjdXkyZMVGBhoYDLzqlSpkgoXLqyzZ89q7969Nvt+/fVXlSpVyqBkQOpwjWwG+OKLL9SlSxf5+/tbr9vYtWuXqlWrptWrV6tTp04GJzSvhQsXaty4cRoyZIgmT56c5JNtTtECcjZnZ2fdvn07yfYTJ06oePHiBiTKOSwWi1q2bKmWLVumOMbPz0+bNm1idlg7nDlzRrVq1Uqy3dnZWTExMQYkyhn279+v4OBgPfbYY6pRo4Yk6cCBA4qLi1OLFi3UsWNH69j169cbFdM0tm7danPf09PT5v6ZM2f0yiuvZGUkwG4U2Qzw5ptvatSoUXr77bdtto8fP15vvvkmRTYd/vpku0OHDpo2bZp1u7+/v4YNG2ZgMgBZoV27dnr77be1du1aSQ/KV2RkpEaMGMHv1ixw9uxZZoS1U7ly5RQeHp5k0qfNmzeratWqBqUyv0KFCiX5/zwfsKTdo2Z+Hzx4sM39zz77TO3atWMyOGQrFNkMcPHiRfXo0SPJ9hdffFEzZswwIFHOwSfbQO42c+ZM/d///Z9KlCihP//8U02aNFFUVJTq16+vyZMnGx0PSCIoKEgDBw7U3bt3lZiYqN27d+uzzz7T1KlTmUQrHT788MNUjQsJCVFsbCzLH2awV199VfXq1WNSTWQrFNkM0LRpU/3888+qUKGCzfYdO3aoUaNGBqXKGfhkG8jd3N3d9f333yskJEQHDhxQdHS0ateurYCAAKOjAcnq27ev8uXLpzFjxujOnTvq1q2bSpUqpblz56pLly5Gx8vxWrdurfDwcApXBmORE2RHFNk0+vrrr61ft2vXTiNGjFBYWJh16vJdu3Zp3bp1mjhxolERcwQ+2QZyr3v37ilfvnwKDw9XgwYN1KBBA6MjAanSvXt3de/eXXfu3FF0dLRKlCiRZExISIj8/f05cpjBKFxA7sE6smnk4JC6CZ8tFot1giKkzaeffqoJEyYoIiJCklSqVClNnDhRL7/8ssHJAGQ2Hx8fffnll9bJXZC1WKM787i5uXHkMBPwM5s5eF+RHbH8TholJCSk6kaJTb/u3bvr5MmTio6OVlRUlC5cuECJBXKJ0aNH66233tL169eNjgJkKI4jAED6cGpxFmIZg/TJnz+/8ufPb3QMAFlo/vz5OnXqlEqVKqWyZcsmmTFz3759BiXLGWJiYrR27VqdOnVKnp6e6tq1q4oWLWrd/8EHH8jDw8PAhAAAJI8im4VYxiB1atWqJYvFkqqx/BEL5GwdOnQwOkKO4uvrqx07dqhIkSI6f/68GjdurBs3bqhSpUqKiIjQO++8o127dqlcuXKSpG7duhmcGLBPav9+gH3Kli2rvHnzGh0DsEGRRbbDH64A/jJ+/HijI+Qox44d0/379yVJo0aNUqlSpRQeHi53d3dFR0frueee0+jRo7Vq1SqDkwJpwynbmePw4cNGRwCSoMgi2+EPVwDIfKGhoVq0aJHc3d0lSQULFtTEiRNZIiaLcOTQfrdu3VJUVJQkqWTJktaf3X/6448/sjqW6RQuXDjVP3/MT4DsjCILU9i7d6+OHj0q6cGpcXXq1DE4EYDMUqRIEZ04cULFihV75B9c/JFlv7/ez7t378rT09NmX+nSpXXlyhUjYuU6HDlMvaVLl2rWrFk6fvy4zfbKlStr6NChTABppzlz5hgdAcgQFFlkaxcuXFDXrl0VEhKiQoUKSZJu3rypp556SqtXr9Zjjz1mbEAAGW727NlydXWVxB9cmaFFixbKkyePbt++rePHj6tatWrWfefOnbOZ7Alpc//+fW3btk0RERHq1q2bXF1d9fvvv8vNzU0FCxaUxJHD1JoxY4YmTJig119/Xa1atbJOPnbp0iVt2bJFgwcP1o0bNzRs2DCDk5pHz549jY4AZAjWkc1CrMFlv2eeeUY3b97UypUrVblyZUnS8ePH1bt3b7m5uWnz5s0GJwQA85g4caLN/SeffFKtWrWy3h8+fLguXLigzz77LKuj5Rjnzp3TM888o8jISMXGxurEiRPy8fHR4MGDFRsbq0WLFhkd0VTKli2rGTNm6IUXXkh2/5o1azR8+HBFRkZmcbKcIyIiQh9++KEiIiI0d+5clShRQt9++63KlCmjxx9/3Oh4QIoosul09epVLV++XKGhoTbXbTz11FPq1auXihcvbh27atUqtW/fPsnyEUhZvnz5tHPnTtWqVctme1hYmBo1aqQ7d+4YlAxAZrl9+3aqx7q5uWViEsB+HTp0kKurq5YtW6aiRYtaP8Detm2b+vXrp5MnTxod0VTy5cunffv2qWrVqsnuP3LkiPz9/fl7II1++ukntW7dWg0aNND27dt19OhR+fj4aNq0adq7d68+//xzoyMCKXIwOoCZ7dmzR5UqVdK8efPk7u6uxo0bq3HjxnJ3d9e8efNUpUoV7d271zq+W7dulFg7eXl5JbtkUXx8vEqVKmVAIgCZrVChQipcuHCqbkB28/PPP2vMmDFycnKy2e7t7a3ffvvNoFTm9cQTT2jatGnW2bb/KT4+Xu+++66eeOIJA5LlDCNHjtSkSZP0/fff2/zMNm/eXLt27TIwGfBoXCObDoMGDdLzzz+vRYsWJZmMJDExUf3799egQYMUGhpqUELzmzFjhgYNGqQFCxbI399f0oOJnwYPHqz33nvP4HQAMsPWrVutX589e1YjR45Ur169VL9+fUkPZttduXKlpk6dalREIEUJCQmKj49Psv3ChQvWa7+RevPnz1erVq1UsmRJNW7c2OYa2e3bt8vJyUlbtmwxOKV5HTp0KNnltkqUKKGrV68akAhIPU4tTod8+fJp//79qlKlSrL7jx07plq1aunPP//M4mQ5R+HChXXnzh3dv39fefI8+Nzlr6//fXSb2UuBnKdFixbq27evunbtarN91apVWrx4sbZt22ZMMCAFnTt3lru7uxYvXixXV1cdPHhQxYsXV/v27VWmTBl9+OGHRkc0nT/++EOffPKJdu3aZXMZV/369dWtWzcuMUiHxx57TGvXrtVTTz1lM5fLl19+qWHDhikiIsLoiECKOCKbDiVLltTu3btTLLK7d++2fnKItGHGUiB3+2ut03/z9/dX3759DUgEPNzMmTPVqlUr+fr66u7du+rWrZtOnjypYsWKMYlWGrm6uuq1117Ta6+9ZnSUHKdLly4aMWKE1q1bJ4vFooSEBIWEhGjYsGHq0aOH0fGAh+KIbDosWLBAQ4cO1auvvqoWLVrYnO4SHBysJUuW6L333tOAAQMMTgoA5lS5cmW1b99e06dPt9n+5ptv6quvvkqyriSQHdy/f19r1qzRgQMHFB0drdq1a6t79+7Kly+f0dFynHv37unixYsqU6aM0VFMKS4uTgMHDtSKFSsUHx+vPHnyKD4+Xt26ddOKFSvk6OhodEQgRRTZdFqzZo1mz56tsLAw6zUxjo6OqlOnjoKCglKcLh72uXz5si5fvqyEhASb7dWrVzcoEYCssGnTJnXq1EkVKlRQvXr1JD042+XkyZP64osv1KZNG4MTAjDSgQMHVLt27WSvS0bqRUZG6vDhw4qOjlatWrVUsWJFoyMBj0SRzSD37t2zXhRfrFgx5c2b1+BEOUNYWJh69uypo0eP6t8/qhaLhX+4gFzg/PnzWrhwoY4dOyZJqlq1qvr37y8vLy+DkwFJrVy5UsWKFdOzzz4r6cHZA4sXL5avr68+++wzlS1b1uCEOQtFFsi9KLLI1mrUqKHy5ctrxIgR8vDwSDI7NH8QAACyk8qVK2vhwoVq3ry5QkND1aJFC82ZM0f/+9//lCdPHq1fv97oiKZSu3bth+7/888/deLECYqsHYKCglI9dtasWZmYBEgfJntCtnb69Gl98cUXqlChgtFRAGSRgwcPqlq1anJwcNDBgwcfOpbLC5DdnD9/3vpv1oYNG/R///d/euWVV9SgQQM1bdrU2HAmdOTIEXXp0kXlypVLdv/Fixd14sSJLE5lbvv377e5v2/fPt2/f1+VK1eWJJ04ccJ6mRyQnVFkka21aNFCBw4coMgCuUjNmjUVFRWlEiVKqGbNmrJYLEkuLZC4vADZU8GCBXXt2jWVKVNGW7ZssR79cnFxYTm+NKhWrZrq1auX4ozF4eHhWrJkSRanMrd/rtU9a9Ysubq6auXKlSpcuLAk6caNG+rdu7caNWpkVEQgVSiyyNaWLl2qnj176vDhw6pWrVqSa4/btWtnUDIAmeXMmTMqXry49WvATJ5++mn17dtXtWrV0okTJ6wTkv3666/y9vY2NpwJNWjQ4KGzk7u6uqpx48ZZmChnmTlzprZs2WItsZJUuHBhTZo0SS1bttTQoUMNTAc8HEUW2VpoaKhCQkL07bffJtnH0RggZ/rnte9cBw+zWbBggcaMGaPz58/riy++UNGiRSU9mLywa9euBqczn7lz5z50f/ny5W2OMMI+t2/f1pUrV5Jsv3Lliv744w8DEgGpx2RPyNa8vb31n//8R2PHjrWu0wsgdzl+/Lj++9//6ujRo5IezFo8aNAg6/VcAPCXAQMG6O2331axYsWMjmIKPXr00M8//6yZM2eqbt26kqRffvlFw4cPV6NGjbRy5UqDEwIpo8giW3N1dVV4eLjKly9vdBQABvjiiy/UpUsX+fv7q379+pKkXbt2ac+ePVq9erU6depkcELA1vbt2x+6n9NgM5ebm5vCw8Pl4+NjdBRTuHPnjoYNG6bly5fr3r17kqQ8efLo5Zdf1owZM1SgQAGDEwIpo8giW+vZs6caNWqkvn37Gh0FgAHKly+v7t276+2337bZPn78eH3yySeKiIgwKBmQPAcHhyTb/rl0HJfEZC5XV1cdOHCAImunmJgY6+/T8uXLU2BhClwji2ytUqVKGjVqlHbs2CE/P78kkz29/vrrBiUDkBUuXryoHj16JNn+4osvasaMGQYkAh7uxo0bNvfv3bun/fv3a+zYsZo8ebJBqYCHK1CggIoUKWL9GjADiiyytaVLl6pgwYL66aef9NNPP9nss1gsFFkgh2vatKl+/vnnJEtw7dixg6UhkC25u7sn2fb000/LyclJQUFBCgsLMyAVkLyEhARNmjRJM2fOVHR0tKQHR7WHDh2q0aNHJ3uGAZBdUGSRrbH0BpD7fP3119av27VrpxEjRigsLExPPvmkpAfXyK5bt04TJ040KiJgNw8Pj4cuIwMYYfTo0Vq2bJmmTZumBg0aSHrwQeGECRN09+5dziJAtsY1sgCAbCW1RwBYggvZ0cGDB23uJyYm6uLFi5o2bZru37+vHTt2GJQsd+AaWfuUKlVKixYtUrt27Wy2f/XVVxowYIB+++03g5IBj8YRWWQ7QUFBeuedd1SgQAEFBQU9dOysWbOyKBWArJKQkGB0BCDNatasKYvFon8fJ3jyySe1fPlyg1KZ2/379zVlyhT16dNHjz322EPHvvjii3Jzc8uiZOZ3/fp1ValSJcn2KlWq6Pr16wYkAlKPI7LIdpo1a6Yvv/xShQoVUrNmzVIcZ7FY9OOPP2ZhMgDZlZ+fnzZt2iQvLy+joyCXO3funM19BwcHFS9eXC4uLgYlyhlcXV116NAheXt7Gx0lR6lXr57q1aunefPm2WwfNGiQ9uzZo127dhmUDHg0iiwAwPQ4nRBmw4cv9mnfvr06duyonj17Gh0lR/npp5/07LPPqkyZMta1ukNDQxUZGalvv/2WSfWQrXFqMUzl9u3b+vHHH1WlSpVkT4UBAMAMzp49q3v37hkdwzRat26tkSNH6tChQ6pTp06SJWL+fY0nUqdJkyY6fvy4Fi5cqKNHj0qSOnbsqAEDBqhUqVIGpwMejiOyyNZeeOEFNW7cWIGBgfrzzz9Vo0YNnT17VomJiVq9erU6depkdEQA2QBHZGE2/Mza52GTwDHxW/rcvXtXBw8e1OXLl5PMUcAHBMjOOCKLbG379u0aPXq0JOnLL79UYmKibt68qZUrV2rSpEkUWQAAcgEmgcscmzdvVo8ePXTt2rUkE5TxAQGyO1Y5RrZ269YtFSlSRNKDX7adOnVS/vz59eyzz+rkyZMGpwMAAFnt7t27RkfIMQYNGqTnn39ev//+uxISEmxulFhkdxRZZGteXl4KDQ1VTEyMNm/erJYtW0qSbty4wQyQQC7E1TBA7hQfH6933nlHpUuXVsGCBXX69GlJ0tixY7Vs2TKD05nXpUuXFBQUJA8PD6OjAHajyCJbGzJkiLp3767HHntMpUqVUtOmTSU9OOXYz8/P2HAAspyzs7N1QpJ/+uCDD/hDDMjBJk+erBUrVmj69OlycnKybq9WrZqWLl1qYDJz+7//+z9t27bN6BhAmjDZE7K9sLAwRUZG6umnn1bBggUlSRs3blShQoXUoEEDg9MByAxBQUHJbp87d65efPFFFS1aVJI0a9asrIwFZJhVq1apffv2SWbfRfIqVKigDz74QC1atLCZKOvYsWOqX7++bty4YXREU7pz546ef/55FS9eXH5+fsqbN6/N/tdff92gZMCjUWSRI7i5uSk8PJzZH4EcwsHBQTVq1FChQoVstv/000/y9/dXgQIFZLFY9OOPPxoTEHiI4OBgzZ4923r2QNWqVTVkyBAFBAQYnMy88uXLp2PHjqls2bI2RfbIkSOqW7euoqOjjY5oSsuWLVP//v3l4uKiokWLymKxWPdZLBbrKdxAdsSsxcgR+DwGyFmmTJmixYsXa+bMmWrevLl1e968ebVixQr5+voamA5I2fvvv6/Bgwfr//7v/zR48GBJ0q5du9SmTRvNnj1bAwcONDihOfn6+urnn39W2bJlbbZ//vnnqlWrlkGpzG/06NGaOHGiRo4c+dAljoDsiCILAMh2Ro4cqRYtWujFF19U27ZtNXXq1CSnvAHZ0ZQpUzR79mwFBgZat73++utq0KCBpkyZQpFNo3Hjxqlnz5767bfflJCQoPXr1+v48eP66KOP9L///c/oeKYVFxenzp07U2JhSvzUAgCypSeeeEJhYWG6cuWK/P39dfjwYZvT3oDs6ObNm3rmmWeSbG/ZsqVu3bplQKKcoX379vrmm2/0ww8/qECBAho3bpyOHj2qb775Rk8//bTR8UyrZ8+eWrNmjdExgDThiCwAINsqWLCgVq5cqdWrVysgIIB1DZHttWvXTl9++aWGDx9us/2rr77Sf/7zH4NS5QyNGjXS999/b3SMHCU+Pl7Tp0/Xd999p+rVqyc584UJ9ZCdUWSRI3CUBsjZunTpooYNGyosLCzJNXKA0ebNm2f92tfXV5MnT9a2bdtUv359SQ+ukQ0JCdHQoUONiggk69ChQ9ZrjA8fPmyzj7+tkN0xazFyhH/OYAgAQFYqV65cqsYxC6x9ChcunOoydf369UxOAyC74YgsTOn8+fMaP368li9fLkn69ttvVbp0aYNTAQByozNnzhgdIUeaM2eO9etr165p0qRJatWqlfVId2hoqL777juNHTvWoIQAjMQRWZjSgQMHVLt2ba6XAwAgF+jUqZOaNWtmMxu0JM2fP18//PCDNmzYYEwwAIahyCJb+vrrrx+6//Tp0xo6dChFFgCQrfTp0+eh+/86kwj2KViwoMLDw1WhQgWb7adOnVLNmjUVHR1tUDIARuHUYmRLHTp0kMVi0cM+Z2ESAgBAdnPjxg2b+/fu3dPhw4d18+ZNNW/e3KBU5le0aFF99dVXSSbM+uqrr1S0aFGDUgEwEkUW2ZKnp6fef/99tW/fPtn94eHhqlOnThanAgDg4b788ssk2xISEvTaa6+pfPnyBiTKGSZOnKi+fftq27ZtqlevniTpl19+0ebNm7VkyRKD0wEwgoPRAYDk1KlTR2FhYSnuf9TRWgAAsgsHBwcFBQVp9uzZRkcxrV69eikkJERubm5av3691q9fLzc3N+3YsUO9evUyOh4AA3BEFtnS8OHDFRMTk+L+ChUqaOvWrVmYCACAtIuIiND9+/eNjmFq9erV06effmp0DADZBJM9AQAAZJCgoCCb+4mJibp48aI2btyonj17av78+QYlM7/4+Hht2LBBR48elSQ9/vjjateunRwdHQ1OBsAIFFkAAIAM0qxZM5v7Dg4OKl68uJo3b64+ffooTx5OhkuLU6dO6dlnn9WFCxdUuXJlSdLx48fl5eWljRs3cv0xkAtRZAEAAJCttWnTRomJifr0009VpEgRSdK1a9f04osvysHBQRs3bjQ4IYCsRpEFAABAtlagQAHt2rVLfn5+NtsPHDigBg0asI4skAsxazEAAEAGuXTpkl566SWVKlVKefLkkaOjo80NaePs7Kw//vgjyfbo6Gg5OTkZkAiA0bhQAwAAIIP06tVLkZGRGjt2rDw9PWWxWIyOlCP85z//0SuvvKJly5apbt26kh6sI9u/f3+1a9fO4HQAjMCpxQAAABnE1dVVP//8s2rWrGl0lBzl5s2b6tmzp7755hvlzZtXknT//n21a9dOK1askLu7u8EJAWQ1jsgCAABkEC8vL3GMIOMVKlRIX331lU6dOmVdfqdq1aqqUKGCwckAGIUjsgAAABlky5Ytmjlzpj744AN5e3sbHQcAciyKLAAAQDoULlzY5lrYmJgY3b9/X/nz57eeBvuX69evZ3W8HKFTp06qW7euRowYYbN9+vTp2rNnj9atW2dQMgBGocgCAACkw8qVK1M9tmfPnpmYJOcqXry4fvzxxyTL7xw6dEgBAQG6dOmSQckAGIVrZAEAANIhLeV02rRp6t+/vwoVKpTxgXKglJbZyZs3r27fvm1AIgBGYx1ZAACALDZlyhROM7aDn5+f1qxZk2T76tWr5evra0AiAEbjiCwAAEAW48ou+4wdO1YdO3ZURESEmjdvLkkKDg7WZ599xvWxQC5FkQUAAEC21rZtW23YsEFTpkzR559/rnz58ql69er64Ycf1KRJE6PjATAAkz0BAABkMVdXVx04cEA+Pj5GRwEAU+KILAAAAEwhLi5Oly9fVkJCgs32MmXKGJQIgFEosgAAAMjWTp48qT59+mjnzp022xMTE2WxWBQfH29QMgBGocgCAACkQ1BQkN555x0VKFBA27dv11NPPaU8eR7+J1ajRo2UL1++LEpofr169VKePHn0v//9T56enrJYLEZHAmAwrpEFAABIh7x58+rChQvy8PCQo6OjLl68qBIlShgdK0cpUKCAwsLCVKVKFaOjAMgmOCILAACQDt7e3po3b55atmypxMREhYaGqnDhwsmObdy4cRanyxl8fX119epVo2MAyEY4IgsAAJAOGzZsUP/+/XX58mVZLJYU14jlWs60+/HHHzVmzBhNmTJFfn5+yps3r81+Nzc3g5IBMApFFgAAIANER0fLzc1Nx48fT/HUYnd39yxOlTM4ODhIUpJrY5nsCci9OLUYAAAgAxQsWFBbt25VuXLlHjnZE+yzdetWoyMAyGY4IgsAAJBBUprs6dq1aypRogRHDgEggzgYHQAAACCnSOn4QGxsrJycnLI4Tc7y888/68UXX9RTTz2l3377TZL08ccfa8eOHQYnA2AEznsBAABIp3nz5kl6cA3n0qVLVbBgQeu++Ph4bd++naVj0uGLL77QSy+9pO7du2vfvn2KjY2VJN26dUtTpkzRpk2bDE4IIKtxajEAAEA6lStXTpJ07tw5PfbYY3J0dLTuc3Jykre3t95++23Vq1fPqIimVqtWLb3xxhvq0aOHXF1ddeDAAfn4+Gj//v1q3bq1oqKijI4IIItxRBYAACCdzpw5I0lq1qyZ1q9fn+I6skib48ePJ7sGr7u7u27evJn1gQAYjmtkAQAAMsjWrVtTVWLd3Nx0+vTpLEiUM5QsWVKnTp1Ksn3Hjh3y8fExIBEAo1FkAQAAshhXdtmnX79+Gjx4sH755RdZLBb9/vvv+vTTTzVs2DC99tprRscDYABOLQYAAEC2NnLkSCUkJKhFixa6c+eOGjduLGdnZw0bNkyDBg0yOh4AAzDZEwAAQBb754RFSL24uDidOnVK0dHR8vX1tZkdWpIuXLigUqVKycGBkw6BnI4jsgAAADAFJycn+fr6prjf19dX4eHhfEAA5AJ8XAUAAJDFLBaL0RFyJE40BHIPiiwAAEAWo3ABQPpQZAEAALLYt99+q9KlSxsdAwBMiyILAACQTvv27dOZM2es9z/++GM1aNBAXl5eatiwoVavXm0zvmHDhnJ2ds7qmACQY1BkAQAA0ql3796KiIiQJC1dulSvvvqq/P39NXr0aD3xxBPq16+fli9fbnDKnI9rj4Hcg1mLAQAA0unkyZOqWLGiJOn999/X3Llz1a9fP+v+J554QpMnT1afPn2MipgrcO0xkHtwRBYAACCd8ufPr6tXr0qSfvvtN9WtW9dmf7169WxOPUbanDp1St99953+/PNPSUmL65EjR1S2bFkjogHIYhRZAACAdGrdurUWLlwoSWrSpIk+//xzm/1r165VhQoVjIiWI1y7dk0BAQGqVKmS2rRpo4sXL0qSXn75ZQ0dOtQ6zsvLS46OjkbFBJCFLImcgwEAAJAuv//+uxo0aKAyZcrI399fCxcuVJ06dVS1alUdP35cu3bt0pdffqk2bdoYHdWUevToocuXL2vp0qWqWrWqDhw4IB8fH3333XcKCgrSr7/+anREAFmMa2QBAADSqVSpUtq/f7+mTZumb775RomJidq9e7fOnz+vBg0aKCQkRP7+/kbHNK0tW7bou+++02OPPWazvWLFijp37pxBqQAYiSILAACQAQoVKqRp06Zp2rRpRkfJcWJiYpQ/f/4k269fv84yRkAuxTWyAAAAyNYaNWqkjz76yHrfYrEoISFB06dPV7NmzQxMBsAoXCMLAACAbO3w4cNq0aKFateurR9//FHt2rXTr7/+quvXryskJETly5c3OiKALEaRBQAAQLZ369YtzZ8/XwcOHFB0dLRq166tgQMHytPT0+hoAAxAkQUAAAAAmArXyAIAACBb27x5s3bs2GG9v2DBAtWsWVPdunXTjRs3DEwGwCgUWQAAAGRrw4cP1+3btyVJhw4dUlBQkNq0aaMzZ84oKCjI4HQAjMDyOwAAAMjWzpw5I19fX0nSF198obZt22rKlCnat2+f2rRpY3A6AEbgiCwAAACyNScnJ925c0eS9MMPP6hly5aSpCJFiliP1ALIXTgiCwAAgGytYcOGCgoKUoMGDbR7926tWbNGknTixAk99thjBqcDYASOyAIAACBbmz9/vvLkyaPPP/9cCxcuVOnSpSVJ3377rZ555hmD0wEwAsvvAAAAAPj/9u4mtK5qDQPwd04SBItaW5taIT9NUkKhMXQmFEEKDowa40wQUbDTjKUV0kYRBNtS6shSRGlBQTIJZqC0YIwDISQRHYgYTUxqUhyZWqslPdl3cLmFpPbSSc9aOz7PKGvvPXiHefnWWgdKxdZiAACytrCw8H/ft7a21ikJkAsTWQAAslatVqNSqdz2fa1Wq2MaIAcmsgAAZG1mZmbdenV1NWZmZuLkyZPx1ltvJUoFpGQiCwBAKY2NjcU777wTX3zxReooQJ25tRgAgFLq7u6OycnJ1DGABGwtBgAga1euXFm3LooilpeX49ixY7Fnz55EqYCUFFkAALK2devWWy57KooiWlpa4uOPP06UCkjJGVkAALI2Pj6+bl2tVmPHjh3R1dUVjY3mMvBvpMgCALApPP3003H27NnYtWtX6ijAXeayJwAANoUvv/wy/vrrr9QxgDpQZAEAACgVRRYAAIBSUWQBAAAoFUUWAACAUlFkAQAAKBVFFgCATeHIkSOxbdu21DGAOvA7sgAAZO+HH36Id999N77//vuIiNi7d28MDg5Gd3d34mRACiayAABkbWRkJPbt2xdTU1PR29sbvb29MT09Hfv27YuRkZHU8YAETGQBAMhaZ2dnvPjii/HGG2+se3706NE4f/58/PTTT4mSAakosgAAZO3ee++Nb7/9Nrq6utY9//HHH6O3tzeuXbuWKBmQiq3FAABk7YknnoiJiYlbnn/11Vfx+OOPJ0gEpNaYOgAAAGw0Ojp68+/+/v547bXXYmpqKh577LGIiPj666/jk08+ieHh4VQRgYRsLQYAIDvV6p1tHKxUKlGr1e5yGiA3iiwAAACl4owsAAAApeKMLAAAWdv4szsbDQ0N1SkJkAtbiwEAyNr+/fvXrVdXV2Nubi4aGxujs7MzpqenEyUDUjGRBQAgazMzM7c8u3LlSrzyyivx/PPPJ0gEpGYiCwBAKX333Xfx7LPPxvz8fOooQJ257AkAgFJaWVmJlZWV1DGABGwtBgAga6dPn163LooilpeX49y5c/HUU08lSgWkZGsxAABZ271797p1tVqNHTt2xMGDB+Pw4cNx3333JUoGpKLIAgAAUCrOyAIAAFAqzsgCAJC1P//8M95+++24ePFi/Pbbb7G2trbu/c8//5woGZCKIgsAQNYOHToU4+Pj8dJLL8WuXbuiUqmkjgQk5owsAABZ27p1a4yNjcWBAwdSRwEy4YwsAABZe/DBB2Pbtm2pYwAZUWQBAMjam2++GUNDQ3Ht2rXUUYBM2FoMAEB29u/fv+4s7OzsbBRFEe3t7dHU1LTu2+np6XrHAxJz2RMAANkZGBhIHQHImIksAACbwkcffRT9/f2xZcuW1FGAu0yRBQBgU7j//vvjm2++iY6OjtRRgLvMZU8AAGwK5jPw76HIAgAAUCqKLAAAAKWiyAIAAFAqiiwAAAClosgCAJCd06dPx99//x0REQsLC3d0kVNbW1s0NTXd7WhABvz8DgAA2WlsbIylpaVobm6OhoaGWF5ejubm5tSxgEw0pg4AAAAbPfLIIzEyMhJ9fX1RFEVcunTp5oR2o9bW1jqnA1IzkQUAIDtnzpyJwcHBuHHjxm2/KYoiKpVK1Gq1OiYDcqDIAgCQpT/++CN++eWXePTRR+PChQuxffv2f/yut7e3zsmA1BRZAACy9uGHH8YLL7wQ99xzT+ooQCbcWgwAQNaGh4fj6tWrtzz//fffo6OjI0EiIDVFFgCArM3Pz//jOdjr16/Hr7/+miARkJpbiwEAyNLo6OjNvz/77LN44IEHbq5rtVpcvHgx2tvbEyQDUnNGFgCALFWr/908WKlUYuO/rE1NTdHe3h4nTpyIZ555JkU8ICFFFgCArO3evTsmJyfjoYceSh0FyIQzsgAAZG1ubu6OSmxPT08sLi7WIRGQmiILAMCmMD8/H6urq6ljAHWgyAIAAFAqiiwAAAClosgCAABQKoosAAAApaLIAgAAUCqKLAAAWRocHIyJiYk7/v69996LnTt33sVEQC4qRVEUqUMAAMBG1Wo1KpVKdHZ2xquvvhovv/xyPPzww6ljARkwkQUAIFuff/559PX1xfHjx6O1tTWee+65+PTTT2NtbS11NCAhRRYAgGz19PTEqVOnYmlpKc6fPx/Xr1+PgYGBaGlpiddffz1mZ2dTRwQSsLUYAIAsVavVuHz5cjQ3N697vrCwEO+//3588MEHsbi4GLVaLVFCIBVFFgCALN2uyP5PURRx4cKFePLJJ+ucDEjN1mIAALLU1tYWDQ0Nt31fqVSUWPiXMpEFAACgVExkAQAAKBVFFgAAgFJRZAEAACgVRRYAAIBSUWQBAAAoFUUWAACAUlFkAQAAKBVFFgAAgFL5Dx+yoXrWbjZtAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Sort model results by F1-scores\n",
        "all_model_results.sort_values(\"f1\", ascending=False)[\"f1\"].plot(kind=\"bar\", figsize=(10,7));"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 825
        },
        "id": "sWhAjRJYdtiG",
        "outputId": "bb4ca496-911a-4bc2-ec7d-e65667f010cf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzoAAAMoCAYAAAAHr5UkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABvDklEQVR4nO3deVxU9cLH8e+AghvgjksoLqWSOySZaZakqTe1vF1NS6W0rDALNfW6l9u1XK+mlZq2mPtS1yWTXJIwDcQlV9zAFNw3TFDg+cOnyQlQloEzc/y8X695Pcw5Z+DrPF2Y7zm/3+9Y0tLS0gQAAAAAJuJidAAAAAAAsDeKDgAAAADToegAAAAAMB2KDgAAAADToegAAAAAMB2KDgAAAADToegAAAAAMB2KDgAAAADTKWB0gKxITU3VqVOn5OHhIYvFYnQcAAAAAAZJS0vT1atXVaFCBbm4ZH7dximKzqlTp+Tj42N0DAAAAAAOIi4uTg888ECm+52i6Hh4eEi6/Y/x9PQ0OA0AAAAAo1y5ckU+Pj7WjpAZpyg6fw5X8/T0pOgAAAAAuOeUFhYjAAAAAGA6FB0AAAAApkPRAQAAAGA6FB0AAAAApkPRAQAAAGA6FB0AAAAApkPRAQAAAGA6FB0AAAAApkPRAQAAAGA6FB0AAAAApkPRAQAAAGA6FB0AAAAApkPRAQAAAGA6FB0AAAAApkPRAQAAAGA6FB0AAAAApkPRAQAAAGA6FB0AAAAApkPRAQAAAGA6FB0AAAAApkPRAQAAAGA6OSo6M2bMkK+vrwoVKqTAwEBt3779rsdPmTJFNWrUUOHCheXj46N3331XN27cyFFgAAAAALiXbBedRYsWKTQ0VCNGjFBUVJTq1aunVq1a6cyZMxkev2DBAg0aNEgjRozQ/v37NWfOHC1atEj//ve/cx0eAAAAADJiSUtLS8vOCwIDA/XII49o+vTpkqTU1FT5+PioT58+GjRoULrjQ0JCtH//foWFhVm39evXT7/88ou2bt2apZ955coVeXl56fLly/L09MxO3LvyHbTabt8rrx0f39boCAAAAIDhstoNsnVFJzk5WZGRkQoKCvrrG7i4KCgoSBERERm+5rHHHlNkZKR1eNvRo0e1Zs0atWnTJtOfk5SUpCtXrtg8AAAAACCrCmTn4HPnziklJUXe3t422729vXXgwIEMX9OlSxedO3dOjz/+uNLS0nTr1i317t37rkPXxo0bp1GjRmUnGgAAAABY5fmqa5s2bdLYsWP18ccfKyoqSsuXL9fq1av1wQcfZPqawYMH6/Lly9ZHXFxcXscEAAAAYCLZuqJTunRpubq6KiEhwWZ7QkKCypUrl+Frhg0bppdfflk9e/aUJNWpU0eJiYl67bXXNGTIELm4pO9a7u7ucnd3z040AAAAALDK1hUdNzc3+fv72ywskJqaqrCwMDVu3DjD11y/fj1dmXF1dZUkZXMdBAAAAADIkmxd0ZGk0NBQde/eXQEBAWrUqJGmTJmixMREBQcHS5K6deumihUraty4cZKkZ599VpMmTVKDBg0UGBiomJgYDRs2TM8++6y18AAAAACAPWW76HTq1Elnz57V8OHDFR8fr/r162vdunXWBQpiY2NtruAMHTpUFotFQ4cO1e+//64yZcro2Wef1ZgxY+z3r4BDYdluAAAAGC3b99ExAvfRca4P5LyvAAAAyCtZ7QbZvqIDwBjOVCAlSiQAADBWni8vDQAAAAD5jaIDAAAAwHQoOgAAAABMh6IDAAAAwHRYjADAfc+ZFnpgkQcAALKGogMAyBMUSACAkRi6BgAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0ChgdAAAAZJ3voNVGR8iy4+PbGh0BwH2MKzoAAAAATIeiAwAAAMB0GLoGAADuewwJBMyHogMAAIA8QYGEkRi6BgAAAMB0KDoAAAAATIeiAwAAAMB0KDoAAAAATIeiAwAAAMB0WHUNAAAAcCKsZpc1XNEBAAAAYDoUHQAAAACmQ9EBAAAAYDoUHQAAAACmQ9EBAAAAYDoUHQAAAACmQ9EBAAAAYDoUHQAAAACmk6OiM2PGDPn6+qpQoUIKDAzU9u3bMz22efPmslgs6R5t2xp38yAAAAAA5pbtorNo0SKFhoZqxIgRioqKUr169dSqVSudOXMmw+OXL1+u06dPWx979+6Vq6urXnjhhVyHBwAAAICMZLvoTJo0Sb169VJwcLD8/Pw0a9YsFSlSRHPnzs3w+JIlS6pcuXLWxw8//KAiRYpQdAAAAADkmWwVneTkZEVGRiooKOivb+DioqCgIEVERGTpe8yZM0edO3dW0aJFMz0mKSlJV65csXkAAAAAQFZlq+icO3dOKSkp8vb2ttnu7e2t+Pj4e75++/bt2rt3r3r27HnX48aNGycvLy/rw8fHJzsxAQAAANzn8nXVtTlz5qhOnTpq1KjRXY8bPHiwLl++bH3ExcXlU0IAAAAAZlAgOweXLl1arq6uSkhIsNmekJCgcuXK3fW1iYmJWrhwod5///17/hx3d3e5u7tnJxoAAAAAWGXrio6bm5v8/f0VFhZm3ZaamqqwsDA1btz4rq9dsmSJkpKS9NJLL+UsKQAAAABkUbau6EhSaGiounfvroCAADVq1EhTpkxRYmKigoODJUndunVTxYoVNW7cOJvXzZkzRx06dFCpUqXskxwAAAAAMpHtotOpUyedPXtWw4cPV3x8vOrXr69169ZZFyiIjY2Vi4vthaKDBw9q69atWr9+vX1SAwAAAMBdZLvoSFJISIhCQkIy3Ldp06Z022rUqKG0tLSc/CgAAAAAyLZ8XXUNAAAAAPIDRQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJgORQcAAACA6VB0AAAAAJhOjorOjBkz5Ovrq0KFCikwMFDbt2+/6/GXLl3SW2+9pfLly8vd3V0PPfSQ1qxZk6PAAAAAAHAvBbL7gkWLFik0NFSzZs1SYGCgpkyZolatWungwYMqW7ZsuuOTk5P19NNPq2zZslq6dKkqVqyoEydOqHjx4vbIDwAAAADpZLvoTJo0Sb169VJwcLAkadasWVq9erXmzp2rQYMGpTt+7ty5unDhgn7++WcVLFhQkuTr65u71AAAAABwF9kaupacnKzIyEgFBQX99Q1cXBQUFKSIiIgMX/Ptt9+qcePGeuutt+Tt7a3atWtr7NixSklJyfTnJCUl6cqVKzYPAAAAAMiqbBWdc+fOKSUlRd7e3jbbvb29FR8fn+Frjh49qqVLlyolJUVr1qzRsGHDNHHiRI0ePTrTnzNu3Dh5eXlZHz4+PtmJCQAAAOA+l+errqWmpqps2bL69NNP5e/vr06dOmnIkCGaNWtWpq8ZPHiwLl++bH3ExcXldUwAAAAAJpKtOTqlS5eWq6urEhISbLYnJCSoXLlyGb6mfPnyKliwoFxdXa3batWqpfj4eCUnJ8vNzS3da9zd3eXu7p6daAAAAABgla0rOm5ubvL391dYWJh1W2pqqsLCwtS4ceMMX9OkSRPFxMQoNTXVuu3QoUMqX758hiUHAAAAAHIr20PXQkND9dlnn2n+/Pnav3+/3njjDSUmJlpXYevWrZsGDx5sPf6NN97QhQsX1LdvXx06dEirV6/W2LFj9dZbb9nvXwEAAAAAd8j28tKdOnXS2bNnNXz4cMXHx6t+/fpat26ddYGC2NhYubj81Z98fHz0/fff691331XdunVVsWJF9e3bVwMHDrTfvwIAAAAA7pDtoiNJISEhCgkJyXDfpk2b0m1r3Lixtm3blpMfBQAAAADZluerrgEAAABAfqPoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADCdHBWdGTNmyNfXV4UKFVJgYKC2b9+e6bHz5s2TxWKxeRQqVCjHgQEAAADgXrJddBYtWqTQ0FCNGDFCUVFRqlevnlq1aqUzZ85k+hpPT0+dPn3a+jhx4kSuQgMAAADA3WS76EyaNEm9evVScHCw/Pz8NGvWLBUpUkRz587N9DUWi0XlypWzPry9vXMVGgAAAADuJltFJzk5WZGRkQoKCvrrG7i4KCgoSBEREZm+7tq1a6pcubJ8fHzUvn17/fbbbzlPDAAAAAD3kK2ic+7cOaWkpKS7IuPt7a34+PgMX1OjRg3NnTtXq1at0ldffaXU1FQ99thjOnnyZKY/JykpSVeuXLF5AAAAAEBW5fmqa40bN1a3bt1Uv359PfHEE1q+fLnKlCmjTz75JNPXjBs3Tl5eXtaHj49PXscEAAAAYCLZKjqlS5eWq6urEhISbLYnJCSoXLlyWfoeBQsWVIMGDRQTE5PpMYMHD9bly5etj7i4uOzEBAAAAHCfy1bRcXNzk7+/v8LCwqzbUlNTFRYWpsaNG2fpe6SkpGjPnj0qX758pse4u7vL09PT5gEAAAAAWVUguy8IDQ1V9+7dFRAQoEaNGmnKlClKTExUcHCwJKlbt26qWLGixo0bJ0l6//339eijj6p69eq6dOmSPvzwQ504cUI9e/a0778EAAAAAP5ftotOp06ddPbsWQ0fPlzx8fGqX7++1q1bZ12gIDY2Vi4uf10ounjxonr16qX4+HiVKFFC/v7++vnnn+Xn52e/fwUAAAAA3CHbRUeSQkJCFBISkuG+TZs22TyfPHmyJk+enJMfAwAAAAA5kuerrgEAAABAfqPoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA06HoAAAAADAdig4AAAAA08lR0ZkxY4Z8fX1VqFAhBQYGavv27Vl63cKFC2WxWNShQ4ec/FgAAAAAyJJsF51FixYpNDRUI0aMUFRUlOrVq6dWrVrpzJkzd33d8ePH1b9/fzVt2jTHYQEAAAAgK7JddCZNmqRevXopODhYfn5+mjVrlooUKaK5c+dm+pqUlBR17dpVo0aNUtWqVXMVGAAAAADuJVtFJzk5WZGRkQoKCvrrG7i4KCgoSBEREZm+7v3331fZsmX16quv5jwpAAAAAGRRgewcfO7cOaWkpMjb29tmu7e3tw4cOJDha7Zu3ao5c+YoOjo6yz8nKSlJSUlJ1udXrlzJTkwAAAAA97k8XXXt6tWrevnll/XZZ5+pdOnSWX7duHHj5OXlZX34+PjkYUoAAAAAZpOtKzqlS5eWq6urEhISbLYnJCSoXLly6Y4/cuSIjh8/rmeffda6LTU19fYPLlBABw8eVLVq1dK9bvDgwQoNDbU+v3LlCmUHAAAAQJZlq+i4ubnJ399fYWFh1iWiU1NTFRYWppCQkHTH16xZU3v27LHZNnToUF29elVTp07NtLy4u7vL3d09O9EAAAAAwCpbRUeSQkND1b17dwUEBKhRo0aaMmWKEhMTFRwcLEnq1q2bKlasqHHjxqlQoUKqXbu2zeuLFy8uSem2AwAAAIC9ZLvodOrUSWfPntXw4cMVHx+v+vXra926ddYFCmJjY+XikqdTfwAAAADgrrJddCQpJCQkw6FqkrRp06a7vnbevHk5+ZEAAAAAkGVcegEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOjkqOjNmzJCvr68KFSqkwMBAbd++PdNjly9froCAABUvXlxFixZV/fr19eWXX+Y4MAAAAADcS7aLzqJFixQaGqoRI0YoKipK9erVU6tWrXTmzJkMjy9ZsqSGDBmiiIgI7d69W8HBwQoODtb333+f6/AAAAAAkJFsF51JkyapV69eCg4Olp+fn2bNmqUiRYpo7ty5GR7fvHlzPffcc6pVq5aqVaumvn37qm7dutq6dWuuwwMAAABARrJVdJKTkxUZGamgoKC/voGLi4KCghQREXHP16elpSksLEwHDx5Us2bNMj0uKSlJV65csXkAAAAAQFZlq+icO3dOKSkp8vb2ttnu7e2t+Pj4TF93+fJlFStWTG5ubmrbtq3++9//6umnn870+HHjxsnLy8v68PHxyU5MAAAAAPe5fFl1zcPDQ9HR0dqxY4fGjBmj0NBQbdq0KdPjBw8erMuXL1sfcXFx+RETAAAAgEkUyM7BpUuXlqurqxISEmy2JyQkqFy5cpm+zsXFRdWrV5ck1a9fX/v379e4cePUvHnzDI93d3eXu7t7dqIBAAAAgFW2rui4ubnJ399fYWFh1m2pqakKCwtT48aNs/x9UlNTlZSUlJ0fDQAAAABZlq0rOpIUGhqq7t27KyAgQI0aNdKUKVOUmJio4OBgSVK3bt1UsWJFjRs3TtLt+TYBAQGqVq2akpKStGbNGn355ZeaOXOmff8lAAAAAPD/sl10OnXqpLNnz2r48OGKj49X/fr1tW7dOusCBbGxsXJx+etCUWJiot58802dPHlShQsXVs2aNfXVV1+pU6dO9vtXAAAAAMAdsl10JCkkJEQhISEZ7vv7IgOjR4/W6NGjc/JjAAAAACBH8mXVNQAAAADITxQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOhQdAAAAAKZD0QEAAABgOjkqOjNmzJCvr68KFSqkwMBAbd++PdNjP/vsMzVt2lQlSpRQiRIlFBQUdNfjAQAAACC3sl10Fi1apNDQUI0YMUJRUVGqV6+eWrVqpTNnzmR4/KZNm/Tiiy9q48aNioiIkI+Pj1q2bKnff/891+EBAAAAICPZLjqTJk1Sr169FBwcLD8/P82aNUtFihTR3LlzMzz+66+/1ptvvqn69eurZs2amj17tlJTUxUWFpbr8AAAAACQkWwVneTkZEVGRiooKOivb+DioqCgIEVERGTpe1y/fl03b95UyZIlMz0mKSlJV65csXkAAAAAQFZlq+icO3dOKSkp8vb2ttnu7e2t+Pj4LH2PgQMHqkKFCjZl6e/GjRsnLy8v68PHxyc7MQEAAADc5/J11bXx48dr4cKFWrFihQoVKpTpcYMHD9bly5etj7i4uHxMCQAAAMDZFcjOwaVLl5arq6sSEhJstickJKhcuXJ3fe1HH32k8ePHa8OGDapbt+5dj3V3d5e7u3t2ogEAAACAVbau6Li5ucnf399mIYE/FxZo3Lhxpq+bMGGCPvjgA61bt04BAQE5TwsAAAAAWZCtKzqSFBoaqu7duysgIECNGjXSlClTlJiYqODgYElSt27dVLFiRY0bN06S9J///EfDhw/XggUL5Ovra53LU6xYMRUrVsyO/xQAAAAAuC3bRadTp046e/ashg8frvj4eNWvX1/r1q2zLlAQGxsrF5e/LhTNnDlTycnJ+uc//2nzfUaMGKGRI0fmLj0AAAAAZCDbRUeSQkJCFBISkuG+TZs22Tw/fvx4Tn4EAAAAAORYvq66BgAAAAD5gaIDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMJ0dFZ8aMGfL19VWhQoUUGBio7du3Z3rsb7/9po4dO8rX11cWi0VTpkzJaVYAAAAAyJJsF51FixYpNDRUI0aMUFRUlOrVq6dWrVrpzJkzGR5//fp1Va1aVePHj1e5cuVyHRgAAAAA7iXbRWfSpEnq1auXgoOD5efnp1mzZqlIkSKaO3duhsc/8sgj+vDDD9W5c2e5u7vnOjAAAAAA3Eu2ik5ycrIiIyMVFBT01zdwcVFQUJAiIiLsFiopKUlXrlyxeQAAAABAVmWr6Jw7d04pKSny9va22e7t7a34+Hi7hRo3bpy8vLysDx8fH7t9bwAAAADm55Crrg0ePFiXL1+2PuLi4oyOBAAAAMCJFMjOwaVLl5arq6sSEhJstickJNh1oQF3d3fm8wAAAADIsWxd0XFzc5O/v7/CwsKs21JTUxUWFqbGjRvbPRwAAAAA5ES2ruhIUmhoqLp3766AgAA1atRIU6ZMUWJiooKDgyVJ3bp1U8WKFTVu3DhJtxcw2Ldvn/Xr33//XdHR0SpWrJiqV69ux38KAAAAANyW7aLTqVMnnT17VsOHD1d8fLzq16+vdevWWRcoiI2NlYvLXxeKTp06pQYNGliff/TRR/roo4/0xBNPaNOmTbn/FwAAAADA32S76EhSSEiIQkJCMtz39/Li6+urtLS0nPwYAAAAAMgRh1x1DQAAAAByg6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHQoOgAAAABMh6IDAAAAwHRyVHRmzJghX19fFSpUSIGBgdq+fftdj1+yZIlq1qypQoUKqU6dOlqzZk2OwgIAAABAVmS76CxatEihoaEaMWKEoqKiVK9ePbVq1UpnzpzJ8Piff/5ZL774ol599VXt3LlTHTp0UIcOHbR3795chwcAAACAjGS76EyaNEm9evVScHCw/Pz8NGvWLBUpUkRz587N8PipU6fqmWee0YABA1SrVi198MEHatiwoaZPn57r8AAAAACQkWwVneTkZEVGRiooKOivb+DioqCgIEVERGT4moiICJvjJalVq1aZHg8AAAAAuVUgOwefO3dOKSkp8vb2ttnu7e2tAwcOZPia+Pj4DI+Pj4/P9OckJSUpKSnJ+vzy5cuSpCtXrmQn7j2lJl236/fLS/b+t+cl3te84Uzvq8R7m1d4X/MG72ve4H3NG7yveYP3NW/kxfv65/dMS0u763HZKjr5Zdy4cRo1alS67T4+PgakcQxeU4xOYE68r3mH9zZv8L7mDd7XvMH7mjd4X/MG72veyMv39erVq/Ly8sp0f7aKTunSpeXq6qqEhASb7QkJCSpXrlyGrylXrly2jpekwYMHKzQ01Po8NTVVFy5cUKlSpWSxWLITOd9duXJFPj4+iouLk6enp9FxTIP3NW/wvuYN3te8wfuaN3hf8wbva97hvc0bzvS+pqWl6erVq6pQocJdj8tW0XFzc5O/v7/CwsLUoUMHSbdLSFhYmEJCQjJ8TePGjRUWFqZ33nnHuu2HH35Q48aNM/057u7ucnd3t9lWvHjx7EQ1nKenp8P/R+KMeF/zBu9r3uB9zRu8r3mD9zVv8L7mHd7bvOEs7+vdruT8KdtD10JDQ9W9e3cFBASoUaNGmjJlihITExUcHCxJ6tatmypWrKhx48ZJkvr27asnnnhCEydOVNu2bbVw4UL9+uuv+vTTT7P7owEAAAAgS7JddDp16qSzZ89q+PDhio+PV/369bVu3TrrggOxsbFycflrMbfHHntMCxYs0NChQ/Xvf/9bDz74oFauXKnatWvb718BAAAAAHfI0WIEISEhmQ5V27RpU7ptL7zwgl544YWc/Cin4+7urhEjRqQbeofc4X3NG7yveYP3NW/wvuYN3te8wfuad3hv84YZ31dL2r3WZQMAAAAAJ5OtG4YCAAAAgDOg6AAAAAAwHYoOAAAAANOh6AAAAAAwHYpOLt28eVOvvPKKjh07ZnQUAAAAAP+PVdfswMvLS9HR0apSpYrRUUwpOTlZx44dU7Vq1VSgQI5WRAcA/M2OHTu0ceNGnTlzRqmpqTb7Jk2aZFAqALAfPjXaQYcOHbRy5Uq9++67RkcxlevXr6tPnz6aP3++JOnQoUOqWrWq+vTpo4oVK2rQoEEGJ3QuEyZMUJ8+fVS4cGFJUnh4uAICAqzr5V+9elUDBw7Uxx9/bGRMp3fkyBF9/vnnOnLkiKZOnaqyZctq7dq1qlSpkh5++GGj4zkdV1dXnT59WmXLlrXZfv78eZUtW1YpKSkGJXNuY8eO1dChQ1WjRg15e3vLYrFY9935Ne7t+eefz/Kxy5cvz8MkQNZ8++23WT62Xbt2eZgk73FFxw5Gjx6tiRMnqkWLFvL391fRokVt9r/99tsGJXNuffv2VXh4uKZMmaJnnnlGu3fvVtWqVbVq1SqNHDlSO3fuNDqiU/n7B0ZPT09FR0eratWqkqSEhARVqFCBD465sHnzZrVu3VpNmjTRli1btH//flWtWlXjx4/Xr7/+qqVLlxod0em4uLgoPj4+XdE5deqUqlWrpj/++MOgZM7N29tb//nPf9SjRw+jozi94ODgLB/7+eef52ESc6tSpcpdS/jRo0fzMY1zc3GxnblisVh0Zx2483129s8EXNGxgzlz5qh48eKKjIxUZGSkzT6LxULRyaGVK1dq0aJFevTRR23+R/fwww/ryJEjBiZzTn8/p8E5DvsbNGiQRo8erdDQUHl4eFi3P/XUU5o+fbqByZzPtGnTJN3+HTp79mwVK1bMui8lJUVbtmxRzZo1jYrn9FxcXNSkSROjY5gC5SV/vPPOOzbPb968qZ07d2rdunUaMGCAMaGc1J1DVTds2KCBAwdq7Nixaty4sSQpIiJCQ4cO1dixY42KaDcUHTtgIYK8cfbs2XRncSUpMTGRoRVwSHv27NGCBQvSbS9btqzOnTtnQCLnNXnyZEm3C/msWbPk6upq3efm5iZfX1/NmjXLqHhO791339WMGTM0ZcoUo6MAWdK3b98Mt8+YMUO//vprPqcxj3feeUezZs3S448/bt3WqlUrFSlSRK+99pr2799vYLrco+jYEZPm7SsgIECrV69Wnz59JP11KXX27NnWsw6AIylevLhOnz6dbmGSnTt3qmLFigalck5/nkB68skntXz5cpUoUcLgRObSv39/tW3bVtWqVZOfn58KFixos5+5JDm3dOlSLV68WLGxsUpOTrbZFxUVZVAq82rdurUGDx7MlbUcOnLkiIoXL55uu5eXl44fP57veeyNT+N2wKT5vDF27Fi1bt1a+/bt061btzR16lTt27dPP//8szZv3mx0PKd05xCgW7duad68eSpdurSk24sRIHc6d+6sgQMHasmSJbJYLEpNTVV4eLj69++vbt26GR3PKW3cuNHoCKb09ttva+PGjXryySdVqlQprpLbybRp0zRkyBD16NFDq1atUnBwsI4cOaIdO3borbfeMjqeKS1dulQlS5Y0OobTeuSRRxQaGqovv/xS3t7ekm7P2R0wYIAaNWpkcLrcYzECO2DSfN45cuSIxo8fr127dunatWtq2LChBg4cqDp16hgdzen4+vpm6cMMQzFzLjk5WW+99ZbmzZunlJQUFShQQCkpKerSpYvmzZtnM/wKWZOSkqJ58+YpLCwsw2WQf/zxR4OSOTcPDw8tXLhQbdu2NTqKqdSsWVMjRozQiy++KA8PD+3atUtVq1bV8OHDdeHCBebq5UKDBg1s/oalpaUpPj5eZ8+e1ccff6zXXnvNwHTOKyYmRs8995wOHTokHx8fSVJcXJwefPBBrVy5UtWrVzc4Ye5QdOygcuXK1knzd/5ii4mJUcOGDXXlyhWjIwLIR3FxcdqzZ4+uXbumBg0a6MEHHzQ6ktMKCQnRvHnz1LZtW5UvXz5dWf9zLg+yp3Llyvr+++9Z0MHOihQpov3796ty5coqW7asfvjhB9WrV0+HDx/Wo48+qvPnzxsd0WmNGjXK5rmLi4vKlCmj5s2b899xLqWlpemHH37QgQMHJEm1atVSUFCQKa70MnTNDpg0n3dSU1MVExOT4ZncZs2aGZQKuDsfHx/5+PgoJSVFe/bs0cWLF5ljkkMLFy7U4sWL1aZNG6OjmMrIkSM1YsQIff755ypSpIjRcUyjXLlyunDhgipXrqxKlSpp27Ztqlevno4dO8ZKl7lw69YtValSRa1atbIOr4L9WCwWtWzZUi1btjQ6it253PsQ3Mufk+b/xKR5+9i2bZuqV6+uWrVqqVmzZmrevLn18eSTTxodz+lERETof//7n822L774QlWqVFHZsmX12muvKSkpyaB05vDOO+9ozpw5km4PuXriiSfUsGFD+fj4aNOmTcaGc1Jubm5OP3TCEU2bNk1r166Vt7e36tSpo4YNG9o8kDNPPfWU9WaMwcHBevfdd/X000+rU6dOeu655wxO57wKFCig3r1768aNG0ZHuW8kJCTo/fffNzpGrnFFxw6YNJ83evfubS2RGQ1ZQfa8//77at68uf7xj39Iur0U8quvvqoePXqoVq1a+vDDD1WhQgWNHDnS2KBObOnSpXrppZckSd99952OHj2qAwcO6Msvv9SQIUMUHh5ucELn069fP02dOlXTp0/nd4AdtW/fnvczD3z66afW0QdvvfWWSpUqpZ9//lnt2rXT66+/bnA659aoUSPt3LlTlStXNjrKfSE+Pl6jRo3S8OHDjY6SK8zRsRMmzdtf0aJFtWvXLs7m2kn58uX13XffKSAgQJI0ZMgQbd68WVu3bpUkLVmyRCNGjNC+ffuMjOnUChUqpJiYGD3wwAN67bXXVKRIEU2ZMkXHjh1TvXr1mK+XA88995w2btyokiVL6uGHH2YZZOA+tXjxYg0ePFjvvvuu/P39VbRoUZv9devWNSiZc9q9e/dd9x84cEAvvviiUlJS8ilR3uCKjp1Uq1ZNn332mdExTCUwMFAxMTEUHTu5ePGizdjmzZs3q3Xr1tbnjzzyiOLi4oyIZhre3t7at2+fypcvr3Xr1mnmzJmSbi9Bz4prOVO8eHGG/OSBqlWraseOHSpVqpTN9kuXLqlhw4Y6evSoQcmc36VLl7R9+/YM55ayzHzOde7cWdLtpdH/ZLFYlJaWJovF4vQfyPNb/fr1re/f3935vjo7ik4OZefMrKenZx4mMa8+ffqoX79+io+PV506ddKdyeXsTfZ4e3vr2LFj8vHxUXJysqKiomxWsbl69Wq69xjZExwcrH/961/WoZZBQUGSpF9++YVVgXKImwDmjePHj2f4wTApKUknT540IJE5fPfdd+ratauuXbsmT09Pmw+KFouFopML3PrAvkqWLKkJEyaoRYsWGe7/7bff9Oyzz+ZzKvuj6ORQ8eLFs9x0OcuQMx07dpQkvfLKK9ZtnL3JuTZt2mjQoEH6z3/+o5UrV6pIkSJq2rSpdf/u3btVrVo1AxM6v5EjR6p27dqKi4vTCy+8IHd3d0mSq6srNw7OhVu3bmnTpk06cuSIunTpIg8PD506dUqenp7WG+Aia/6cKC9J33//vby8vKzPU1JSFBYWpipVqhgRzRT69eunV155RWPHjmU1Oztjbo59+fv769SpU5m+r5cuXTLFSoHM0cmhOxcZOH78uAYNGqQePXpYV1mLiIjQ/PnzNW7cOHXv3t2omE7txIkTd93PL73sOXfunJ5//nlt3bpVxYoV0/z5822GBLVo0UKPPvqoxowZY2BKwNaJEyf0zDPPKDY2VklJSTp06JCqVq2qvn37KikpSbNmzTI6olNxcbm92GpGQ1YKFiwoX19fTZw40bpoCbKnaNGi2rNnj6pWrWp0FNO5s6TfyWKxqFChQqpevTolPRtWrFihxMRE6wI6f3fx4kV9++23Tv8ZlqJjBy1atFDPnj314osv2mxfsGCBPv30U5aVhUO5fPmyihUrlm7OyIULF1SsWDG5ubkZlMwcwsLCFBYWluH4/Llz5xqUynl16NBBHh4emjNnjkqVKmW9IfOmTZvUq1cvHT582OiITqlKlSrasWOHSpcubXQUU3n++efVuXNn/etf/zI6ium4uLhkWNDvHOnx+OOPa+XKldy3DFYMXbODiIiIDM8qBgQEqGfPngYkcl7ffvutWrdurYIFC2Z69uZP7dq1y6dU5nLnUJU7lSxZMp+TmM+oUaP0/vvvKyAggCXR7eSnn37Szz//nK6A+/r66vfffzcolfPLaL7DpUuXVLx48fwPYyJt27bVgAEDtG/fvgznlvJ3K+d++OEHDRkyRGPGjFGjRo0kSdu3b9ewYcM0dOhQeXl56fXXX1f//v2t9zPDvW3dulWPP/640THyDFd07KBGjRpq3769JkyYYLP9vffe06pVq3Tw4EGDkjkfFxcXxcfHq2zZstYhFhlhjk72NWjQIMMP3l5eXnrooYf0zjvvqFatWgYkM4/y5ctrwoQJevnll42OYholSpRQeHi4/Pz85OHhYb2is3XrVnXs2FEJCQlGR3RK//nPf+Tr66tOnTpJkl544QUtW7ZM5cuX15o1a1SvXj2DEzon/m7lndq1a+vTTz/VY489ZrM9PDxcr732mn777Tdt2LBBr7zyimJjYw1K6Xzc3NxUsWJFvfjii3rppZfk5+dndCS74oqOHUyePFkdO3bU2rVrFRgYKOn2WYbDhw9r2bJlBqdzLncO9fn7sB/kTocOHTLcfunSJUVFRal+/fr68ccf1aRJk/wNZiLJycnp/ggjd1q2bKkpU6bo008/lXT7w+K1a9c0YsQItWnTxuB0zmvWrFn6+uuvJd0+U75hwwatW7dOixcv1oABA7R+/XqDEzon/m7lnSNHjmS4iq2np6d1OfQHH3xQ586dy+9oTu3UqVNauHChvvnmG40fP15169ZV165d9eKLL+qBBx4wOl6ucUXHTk6ePKmPP/5YBw4ckCTVqlVLvXv3lo+Pj8HJgKwZMmSItm3bprCwMKOjOK2BAweqWLFiGjZsmNFRTOPkyZNq1aqV0tLSdPjwYQUEBOjw4cMqXbq0tmzZorJlyxod0SkVLlxYhw4dko+Pj/r27asbN27ok08+0aFDhxQYGKiLFy8aHRGw8fjjj8vDw0NffPGFypQpI0k6e/asunXrpsTERG3ZskUbNmzQW2+9xUiaHDp27JgWLFigb775RgcOHFCzZs30448/Gh0rVyg6cCjTpk3L8rF33jQMuffbb7/pySef1JkzZ4yO4rT69u2rL774QnXr1lXdunXTjc+fNGmSQcmc261bt7Ro0SLt2rVL165dU8OGDdW1a1cVLlzY6GhOq0KFClq6dKkee+wx1ahRQ6NHj9YLL7yggwcP6pFHHsnWveJga/Pmzfroo4+0f/9+SZKfn58GDBhgs5w/su/gwYNq37699X5wkhQXF6eqVatq1apVeuihh7Ry5UpdvXqV4cO5kJKSorVr12rYsGHavXu30w+3pOjYyaVLlzRnzhzrL7aHH35Yr7zySqYTv5GxrC4NabFYuHO3nR04cECPP/44l/1z4cknn8x0n8VicfozYzCPkJAQ/e9//9ODDz6onTt36vjx4ypWrJgWLlyoCRMmKCoqyuiITumrr75ScHCwnn/+eesw4PDwcK1YsULz5s1Tly5dDE7o3FJTU7V+/XodOnRI0u050k8//fRd50Yha8LDw/X1119r6dKlunHjhtq3b6+uXbvqmWeeMTparlB07ODXX39Vq1atVLhwYetKIDt27NAff/yh9evXq2HDhgYnBO5t7NixWrdunbZs2WJ0FMBq3Lhx8vb2trlxsHR7qe6zZ89q4MCBBiVzbjdv3tTUqVMVFxenHj16qEGDBpJuzzn18PBgxdAcqlWrll577TW9++67NtsnTZqkzz77zHoyFHmnTp06WrNmDVMHsmjw4MFauHChTp06paefflpdu3ZV+/btTXPDW4qOHTRt2lTVq1fXZ599pgIFbq/vcOvWLfXs2VNHjx7lg2MuJScn69ixY6pWrZr1/UX2ZTYs8PLly4qMjNTq1au1du1aBQUF5XMy84mJidGRI0fUrFkzFS5c2HqPB2Sfr6+vFixYkG6Rh19++UWdO3fOcJlkwCju7u767bffVL16dZvtMTExql27tm7cuGFQsvvHnasz4t6aNGmirl276l//+pcp76vFp0Y7+PXXX21KjiQVKFBA7733ngICAgxM5tyuX7+uPn36aP78+ZJkvSN6nz59VLFiRQ0aNMjghM5l8uTJGW739PRUjRo1tGXLFjVu3DifU5nL+fPn9a9//UsbN26UxWLR4cOHVbVqVb366qsqUaKEJk6caHREpxMfH6/y5cun216mTBmdPn3agETO6173JrsT93vJGR8fH4WFhaUrOhs2bOAKAxxSeHi40RHyFEXHDjw9PRUbG6uaNWvabI+Li5OHh4dBqZzf4MGDtWvXLm3atMlmjGhQUJBGjhxJ0ckmznznvXfffVcFCxZUbGyszT2JOnXqpNDQUIpODvj4+Cg8PDzd/L3w8HBVqFDBoFTOKbMl5v+O+73kXL9+/fT2228rOjraehUyPDxc8+bN09SpUw1OB2Ts8OHD2rhxo86cOZNuifThw4cblMo+KDp20KlTJ7366qv66KOPbH6xDRgwQC+++KLB6ZzXypUrtWjRIj366KM2w34efvhhHTlyxMBk9wdPT09FR0dz+T8b1q9fr++//z7dvQcefPBBnThxwqBUzq1Xr1565513dPPmTT311FOSpLCwML333nvq16+fwemcC/d4yXtvvPGGypUrp4kTJ2rx4sWSbs/bWbRokdq3b29wOiC9zz77TG+88YZKly6tcuXK2XzeslgsFB1IH330kSwWi7p166Zbt25JkgoWLKg33nhD48ePNzid8zp79myG98hITExkvkM+YPpe9iUmJmY4gfPChQtyd3c3IJHzGzBggM6fP68333xTycnJkqRChQpp4MCBGjx4sMHpzI+J3dn33HPP6bnnnjM6BpAlo0eP1pgxY0y7sAvr8dmBm5ubpk6dqosXLyo6OlrR0dG6cOGCJk+ezIebXAgICNDq1autz/8sN7Nnz2YuCRxS06ZN9cUXX1ifWywWpaamasKECXddehoZS0lJ0U8//aRBgwbp7Nmz2rZtm3bt2qULFy44/VlGZ3H8+HHdvHnT6BgA8sjFixf1wgsvGB0jz3BFxw4uX76slJQUlSxZUnXq1LFuv3DhggoUKCBPT08D0zmvsWPHqnXr1tq3b59u3bqlqVOnat++ffr555+1efNmo+MB6UyYMEEtWrTQr7/+quTkZL333nv67bffdOHCBdNP+MwLrq6uatmypfbv368qVarokUceMToSkE7JkiV16NAhlS5dWiVKlLjriIMLFy7kYzLz2L9/v7Zt26bGjRurZs2aOnDggKZOnaqkpCS99NJL1mGtkvTJJ5/I29vbwLTO5YUXXtD69evVu3dvo6PkCYqOHXTu3FnPPvus3nzzTZvtixcv1rfffqs1a9YYlMy5Pf7444qOjtb48eNVp04d6z2JIiIibAol4Chq166tQ4cOafr06fLw8NC1a9f0/PPP66233spw5TDcW+3atXX06NEs30wYyG9/3nvoz68ZWm1f69atU/v27VWsWDFdv35dK1asULdu3VSvXj2lpqaqZcuWWr9+vbXscFPW7KlevbqGDRumbdu2qU6dOipYsKDN/rffftugZPbBfXTsoGTJkgoPD7dZZUm6faf5Jk2a6Pz58wYlA3KOxQjgCNatW6fBgwfrgw8+kL+/v4oWLWqznyvmeYt7ksBojz32mJ566imNHj1aCxcu1Jtvvqk33nhDY8aMkXR7hdbIyEitX7/e4KTO6W4nkSwWi44ePZqPaeyPomMHRYsWtTbhO+3Zs0eBgYG6fv26QcmcW1RUlAoWLGh9X1etWqXPP/9cfn5+GjlypNzc3AxOaG58wMma3bt3Z/nYunXr5mESc3Jx+Wsq6Z1nyv+8CSvLIOctfg9kj6urq06fPp1uIZ3z58+rbNmy/PeaA15eXoqMjFT16tWVmpoqd3d3bd++XQ0aNJAk7d27V0FBQYqPjzc4KRwRQ9fsoFGjRvr000/13//+12b7rFmz5O/vb1Aq5/f6669r0KBBqlOnjo4ePapOnTrp+eef15IlS3T9+nVNmTLF6IimtnbtWlWsWNHoGA6vfv36slgs1g/ef/rzHNKd2/iQk30bN240OgKQZZmdO05KSuLkXC78+XvUxcVFhQoVkpeXl3Wfh4eHLl++bFQ0U8no75azo+jYwejRoxUUFKRdu3apRYsWkm7f52HHjh1cSs2FQ4cOqX79+pKkJUuW6IknntCCBQsUHh6uzp07U3TsLC4uTiNGjNDcuXMl3Z4jhXu780asO3fuVP/+/TVgwADryoARERGaOHGiJkyYYFREp/bEE08YHeG+xsTurJk2bZqk2x8QZ8+erWLFiln3paSkaMuWLeluKo6s8fX11eHDh1WtWjVJt3+nVqpUybo/NjaWOZC59MUXX+jDDz/U4cOHJUkPPfSQBgwYoJdfftngZLlH0bGDJk2aKCIiQh9++KEWL16swoULq27dupozZ44efPBBo+M5rbS0NOsN7jZs2KB//OMfkm7fKf3cuXNGRjOlCxcuaP78+daig6ypXLmy9esXXnhB06ZNU5s2bazb6tatKx8fHw0bNizLd6aHrZ9++kmffPKJjh49qiVLlqhixYr68ssvVaVKFQp5Dvzxxx+KjIxUyZIl5efnZ7Pvxo0bWrx4sbp16yaJid1ZNXnyZEm3/27NmjVLrq6u1n1ubm7y9fXVrFmzjIrn1N544w2bq+G1a9e22b927VqbVdeQPZMmTdKwYcMUEhKiJk2aSJK2bt2q3r1769y5c3r33XcNTpg7zNGBw3rqqafk4+OjoKAgvfrqq9q3b5+qV6+uzZs3q3v37jp+/LjREZ3Kt99+e9f9R48eVb9+/RhelQuFCxdWVFRUuoVJ9u/fr4YNG+qPP/4wKJnzWrZsmV5++WV17dpVX375pfbt26eqVatq+vTpWrNmDataZtOhQ4fUsmVLxcbGymKx6PHHH9fChQutZ8QTEhJUoUIFfg/k0JNPPqnly5erRIkSRkcBsqRKlSoaNWqU9eTGn+bPn6+RI0fajFpwRhQdO0lNTVVMTIzOnDljvQrxp2bNmhmUyrnt3r1bXbt2VWxsrEJDQzVixAhJUp8+fXT+/HktWLDA4ITOxcXFxTqXJDNM7s6dhg0bqnbt2po9e7Z1PH5ycrJ69uypvXv3KioqyuCEzqdBgwZ699131a1bN5uJ8Tt37lTr1q2ZgJxNzz33nG7evKl58+bp0qVLeuedd7Rv3z5t2rRJlSpVougA95lChQpp7969ql69us32w4cPq06dOrpx44ZByeyDomMH27ZtU5cuXXTixIl0HyL54Gh/N27ckKura7q13nF3FStW1Mcff6z27dtnuD86Olr+/v7895oL27dv17PPPqu0tDTrCmu7d++WxWLRd999p0aNGhmc0PkUKVJE+/btk6+vr03ROXr0qPz8/Jz+j3B+8/b21oYNG6yrWaalpenNN9/UmjVrtHHjRhUtWpSikwsdO3ZUo0aNNHDgQJvtEyZM0I4dO7RkyRKDkgEZq127trp06aJ///vfNttHjx6tRYsWac+ePQYlsw/m6NhB7969FRAQoNWrV6t8+fKmWq3CERUqVMjoCE7J399fkZGRmRade13twb01atRIR48e1ddff60DBw5Ikjp16qQuXbqku/8LsqZcuXKKiYmRr6+vzfatW7ey5HEO/PHHHypQ4K8//RaLRTNnzlRISIh1wRfk3JYtWzRy5Mh021u3bq2JEyfmfyDgHkaNGqVOnTppy5Yt1jk64eHhCgsL0+LFiw1Ol3sUHTs4fPiwli5dmu6yH3InJSVFkydP1uLFixUbG6vk5GSb/RcuXDAomXMaMGCAEhMTM91fvXp1lvK1g6JFi+q111676zFt27bV7NmzWSkoC3r16qW+fftq7ty5slgsOnXqlCIiItS/f38NGzbM6HhOp2bNmvr111/TzSObPn26JKldu3ZGxDKNa9euZbiMdMGCBXXlyhUDEgF317FjR/3yyy+aPHmyVq5cKUmqVauWzb2KnBlFxw4CAwMVExND0bGzUaNGafbs2erXr5+GDh2qIUOG6Pjx41q5cqWGDx9udDyn07Rp07vuL1q0KEv55pMtW7awMEEWDRo0SKmpqWrRooWuX7+uZs2ayd3dXf3791efPn2Mjud0nnvuOX3zzTcZLhs7ffp0paamsjpYLtSpU0eLFi1K9zdq4cKF6Va4AxyFv7+/vvrqK6Nj5Anm6NjBihUrNHToUA0YMEB16tRJN3eEu6HnTLVq1TRt2jS1bdtWHh4eio6Otm7btm0bQyzgtLjbfPYlJycrJiZG165dk5+fn819SgBH8d133+n5559Xly5drEseh4WF6ZtvvtGSJUtYYh4OZ82aNXJ1dVWrVq1stn///fdKTU1V69atDUpmHxQdO3BxcUm37c47pTOpM2eKFi2q/fv3q1KlSipfvrxWr16thg0b6ujRo2rQoAF3Qs6mM2fOqGzZstbn0dHRmjx5smJiYlS+fHmFhISoefPmxgW8j1B0ciYuLk7S7XtpAY5q9erVGjt2rKKjo6331RsxYgRXzOGQ6tatq/Hjx9vc/02S1q1bp4EDB2rXrl0GJbOP9J/QkW3Hjh1L9zh69Kj1/yJnHnjgAZ0+fVrS7as769evlyTt2LFD7u7uRkZzSuXLl9eZM2ckST///LMaNWqkEydOqEmTJrpy5YqefvppbdmyxeCUgK1bt25p2LBh8vLykq+vr3x9feXl5aWhQ4fq5s2bRscD0mnbtq3Cw8OVmJioc+fO6ccff6TkwGEdPnw4w2GVNWvWVExMjAGJ7Is5OnZw553RYT/PPfecwsLCFBgYqD59+uill17SnDlzFBsb6/R36jXCnRdvR44cqZdffllz5syxbnvnnXc0atQohYWFGREPyFCfPn20fPlyTZgwQY0bN5YkRUREaOTIkTp//rxmzpxpcELA1qVLl7R06VIdPXpU/fv3V8mSJRUVFSVvb29VrFjR6HiADS8vLx09ejTdypYxMTGmWC2UoWt28uWXX2rWrFk6duyYIiIiVLlyZU2ZMkVVqlTJdDlfZE9ERIQiIiL04IMP6tlnnzU6jtNxcXFRfHy8ypYtqwoVKmj58uV69NFHrft/++03NW/eXGfPnjUw5f2BoWtZ5+XlpYULF6YbJ75mzRq9+OKLDGGFQ9m9e7eCgoLk5eWl48eP6+DBg6pataqGDh2q2NhYffHFF0ZHBGy8/vrrioiI0IoVK1StWjVJt0tOx44d9cgjj2j27NkGJ8wdhq7ZwcyZMxUaGqo2bdro0qVL1jk5xYsX15QpU4wNZyKNGzdWaGgoJScXrl69qitXrqhQoULphv8VKlRI169fNyjZ/eXf//63SpYsaXQMp+Du7p7uTKMkValSJcNlfAEjhYaGqkePHjp8+LDNPd/atGnD0GA4pAkTJqho0aKqWbOmqlSpoipVqqhWrVoqVaqUPvroI6Pj5RpFxw7++9//6rPPPtOQIUPk6upq3R4QEOD0d5Q12sGDBxUSEqIWLVqoRYsWCgkJ0cGDB42O5bQeeughlShRQsePH9evv/5qs++3335ThQoVDEpmHl9++aWaNGmiChUq6MSJE5KkKVOmaNWqVdZjBg8erOLFixuU0LmEhITogw8+UFJSknVbUlKSxowZo5CQEAOTAent2LFDr7/+errtFStWVHx8vAGJgLvz8vLSzz//rNWrV+vNN99Uv379FBYWph9//NEUf6eYo2MHx44dy/CmSu7u7ne9QSPubtmyZercubMCAgKsY/O3bdum2rVra+HCherYsaPBCZ3L328G+vebVR47duyeN7rE3c2cOVPDhw/XO++8ozFjxqS7ussw1uzbuXOnwsLC9MADD6hevXqSpF27dik5OVktWrTQ888/bz12+fLlRsUEJN3+u5/RjUEPHTqkMmXKGJAIuDeLxaKWLVuqZcuWmR5Tp04drVmzxulWvaTo2EGVKlUUHR2dblGCdevWpbv7NLLuvffe0+DBg/X+++/bbB8xYoTee+89ik423WvVn759+9o8/+abb9SuXTtTTEbML39e3e3QoYPGjx9v3R4QEKD+/fsbmMx5FS9ePN3/1p3tDy3uH+3atdP777+vxYsXS7r9ATI2NlYDBw7kbxac2vHjx51ypUuKjh2Ehobqrbfe0o0bN5SWlqbt27frm2++0bhx45x+EpeRTp8+rW7duqXb/tJLL+nDDz80INH95fXXX1dgYCAT5rOBq7v29/nnn2fpuPDwcCUlJbH0PAw1ceJE/fOf/1TZsmX1xx9/6IknnlB8fLwaN26sMWPGGB0PuO9QdOygZ8+eKly4sIYOHarr16+rS5cuqlChgqZOnarOnTsbHc9pNW/eXD/99JOqV69us33r1q1q2rSpQanuHyzImH1c3TVO69atFR0dTTGHoby8vPTDDz8oPDxcu3bt0rVr19SwYUMFBQUZHQ24L1F07KRr167q2rWrrl+/rmvXrtncgf5P4eHhCggI4IzjXXz77bfWr9u1a6eBAwcqMjLSugzytm3btGTJEo0aNcqoiECmuLprHIo5jHbz5k0VLlxY0dHRatKkiZo0aWJ0JOC+x3108pGnpydnHO/BxSVrCwFaLBbrRG/kDe71kjNff/21Ro4cqSNHjkiSKlSooFGjRunVV181OJm58d8rHEHVqlW1YsUK68IZgFk46+9YlpfOR3TKe0tNTc3Sg5IDR9W1a1cdPnxY165dU3x8vE6ePEnJAe4TQ4YM0b///W9duHDB6CgAxNA1mICzLnkIcytSpIiKFClidAwA+Wj69OmKiYlRhQoVVLly5XSrVkZFRRmUDLi7xMRELV68WDExMSpfvrxefPFFlSpVyrr/k08+kbe3t4EJc4aiA6fnrEseOrrKlSurYMGCRsdweA0aNJDFYsnSsXzIyTtZ/f8BkJc6dOhgdAQgS/z8/LR161aVLFlScXFxatasmS5evKiHHnpIR44c0QcffKBt27apSpUqkqQuXboYnDhnKDoAMrR3716jIzgFPtg4BoYGwxGMGDHC6AhAlhw4cEC3bt2SJA0ePFgVKlRQdHS0vLy8dO3aNT333HMaMmSIFixYYHDS3KHo5CPOOMIoJUqUyPJ/f4wtzx4+2OS9y5cvKz4+XpJUrlw5eXl5pTvm6tWr+R0LAEwhIiJCs2bNsv5uLVasmEaNGmWKW6RQdPIRZxxhlClTphgd4b7y66+/av/+/ZJuDw/w9/c3OJFzmj17tiZNmqSDBw/abK9Ro4b69evHIg9wCCVLltShQ4dUunTpe55U4kQSHMmf/63euHFD5cuXt9lXsWJFnT171ohYdkXRsZNbt25p06ZNOnLkiLp06SIPDw+dOnVKnp6eKlasmCTOOMI43bt3NzrCfeHkyZN68cUXFR4eruLFi0uSLl26pMcee0wLFy7UAw88YGxAJ/Lhhx9q5MiRevvtt9WqVSvrJNiEhAStX79effv21cWLF9W/f3+Dk+J+N3nyZHl4eEjipBKcS4sWLVSgQAFduXJFBw8eVO3ata37Tpw4YbMYgbPiPjp2cOLECT3zzDOKjY1VUlKSDh06pKpVq6pv375KSkrSrFmzjI5oas66truRjhw5os8//1xHjhzR1KlTVbZsWa1du1aVKlXSww8/bHQ8p/XMM8/o0qVLmj9/vmrUqCFJOnjwoIKDg+Xp6al169YZnNB5VK5cWR9++KH+9a9/Zbh/0aJFGjBggGJjY/M5GQA4v7/feP3RRx9Vq1atrM8HDBigkydP6ptvvsnvaHZF0bGDDh06yMPDQ3PmzFGpUqWsH7o3bdqkXr166fDhw0ZHdDrnzp3T3LlzFRERYTM2/7HHHlOPHj1UpkwZ67ELFixQ+/bt0y3jiYxt3rxZrVu3VpMmTbRlyxbt379fVatW1fjx4/Xrr79q6dKlRkd0WoULF9bPP/+sBg0a2GyPjIxU06ZNdf36dYOSOZ/ChQsrKipKtWrVynD/vn37FBAQwHsKw125ciXLx3p6euZhEgB/xw1D7eCnn37S0KFD5ebmZrPd19dXv//+u0GpnNeOHTv00EMPadq0afLy8lKzZs3UrFkzeXl5adq0aapZs6Z+/fVX6/FdunSh5GTDoEGDNHr0aP3www82/80+9dRT2rZtm4HJnJ+Pj0+GS52npKSoQoUKBiRyXo888ojGjx9vXRXoTikpKfrPf/6jRx55xIBkgK3ixYurRIkSWXoAyF/M0bGD1NRUpaSkpNt+8uRJ67hdZF2fPn30wgsvaNasWekmdaalpal3797q06ePIiIiDEro3Pbs2ZPhcpFly5bVuXPnDEhkHh9++KH69OmjGTNmKCAgQNLthQn69u2rjz76yOB0zmX69Olq1aqVypUrp2bNmtnM0dmyZYvc3Ny0fv16g1MC0saNG61fHz9+XIMGDVKPHj3UuHFjSbdXtJo/f77GjRtnVETgvsXQNTvo1KmTvLy89Omnn8rDw0O7d+9WmTJl1L59e1WqVEmff/650RGdSuHChbVz507VrFkzw/0HDhxQgwYN9Mcff+RzMnN44IEHtHjxYj322GM285tWrFih/v3768iRI0ZHdFolSpTQ9evXdevWLRUocPs80p9f//2qI6sv3dvVq1f11Vdfadu2bTZDWBs3bqwuXbowDAgOp0WLFurZs6defPFFm+0LFizQp59+qk2bNhkTDLhPcUXHDiZOnKhWrVrJz89PN27cUJcuXXT48GGVLl3a6SdxGaFcuXLavn17pkVn+/bt1rO7yL7OnTtr4MCBWrJkiSwWi1JTUxUeHq7+/furW7duRsdzaqy4ZF8eHh5644039MYbbxgdBciSP+9H8ncBAQHq2bOnAYmA+xtXdOzk1q1bWrRokXbt2qVr166pYcOG6tq1qwoXLmx0NKczY8YM9evXT6+//rpatGhhM2QlLCxMn332mT766CO9+eabBid1TsnJyXrrrbc0b948paSkqECBAkpJSVGXLl00b948ubq6Gh0RuKebN2/q9OnTqlSpktFRAKsaNWqoffv2mjBhgs329957T6tWrUp3TygAeYuiA4e0aNEiTZ48WZGRkdb5T66urvL391doaGimS84i62JjY7V3715du3ZNDRo00IMPPmh0JNM4c+aMzpw5o9TUVJvtdevWNSiR+ezatUsNGzbMcH4kYJQ1a9aoY8eOql69ugIDAyXdHoVw+PBhLVu2TG3atDE4IXB/oejYwfz581W6dGm1bdtW0u0zN59++qn8/Pz0zTffqHLlygYndF43b960TpAvXbq0ChYsaHAiIHORkZHq3r279u/fr7//arVYLHwotyOKDhxVXFycZs6cqQMHDkiSatWqpd69e8vHx8fgZMD9h6JjBzVq1NDMmTP11FNPKSIiQi1atNCUKVP0v//9TwUKFNDy5cuNjoj7XGhoaJaPnTRpUh4mMbd69eqpWrVqGjhwoLy9vdOtGshJj6xr2LDhXff/8ccfOnToEEUHAJApFiOwg7i4OFWvXl2StHLlSv3zn//Ua6+9piZNmqh58+bGhgMk7dy50+Z5VFSUbt26pRo1akiSDh06ZB0aiJw7evSoli1bZv19gJzbt2+fOnfurCpVqmS4//Tp0zp06FA+pwLS2717t2rXri0XFxft3r37rscyfBXIXxQdOyhWrJjOnz+vSpUqaf369daz54UKFWIJZDiEO+/zMGnSJHl4eGj+/PnWG9hdvHhRwcHBatq0qVERTaFFixbatWsXRccOateurcDAwExXXIuOjtZnn32Wz6mA9OrXr6/4+HiVLVtW9evXl8ViSTd0VWL4KmAEio4dPP300+rZs6caNGigQ4cOWScb/vbbb/L19TU2HPA3EydO1Pr1623u0l2iRAmNHj1aLVu2VL9+/QxM59xmz56t7t27a+/evapdu3a6OWXt2rUzKJnzadKkyV1XqPLw8FCzZs3yMRGQsWPHjqlMmTLWrwE4DoqOHcyYMUNDhw5VXFycli1bplKlSkm6PTH57zcNA4x25coVnT17Nt32s2fP6urVqwYkMo+IiAiFh4dr7dq16fZxNjd7pk6detf91apVs7lSCRjlzrl3zMMDHAuLEQD3mW7duumnn37SxIkT1ahRI0nSL7/8ogEDBqhp06aaP3++wQmdl6+vr/7xj39o2LBh3NQ2n7355pt6//33Vbp0aaOj4D538OBB/fe//9X+/fsl3V51rU+fPtY5kQDyD0XHDrZs2XLX/QyvgCO5fv26+vfvr7lz5+rmzZuSpAIFCujVV1/Vhx9+qKJFixqc0Hl5eHgoOjpa1apVMzrKfcfT01PR0dGqWrWq0VFwH1u2bJk6d+6sgIAANW7cWJK0bds27dixQwsXLlTHjh0NTgjcXyg6duDi4pJu253LyjJcBY4oMTFRR44ckXR7GBAFJ/e6d++upk2bqmfPnkZHue94eHho165dFB0Yqlq1auratavef/99m+0jRozQV199Zf2dCyB/MEfHDi5evGjz/ObNm9q5c6eGDRumMWPGGJQKuLuiRYuqZMmS1q+Rew899JAGDx6srVu3qk6dOukWI3j77bcNSgYgP5w+fVrdunVLt/2ll17Shx9+aEAi4P5G0bEDLy+vdNuefvppubm5KTQ0VJGRkQakAjKWmpqq0aNHa+LEibp27Zqk22fD+/XrpyFDhmR4hRJZM3v2bBUrVkybN2/W5s2bbfZZLBaKDmByzZs3108//ZRuifmtW7eyfD9gAIpOHvL29r7r8qiAEYYMGaI5c+Zo/PjxatKkiaTbf4RHjhypGzducBUyF1haFrj/fPvtt9av27Vrp4EDByoyMlKPPvqopNtzdJYsWaJRo0YZFRG4bzFHxw7+fifktLQ0nT59WuPHj9etW7e0detWg5IB6VWoUEGzZs1Kd0+XVatW6c0339Tvv/9uUDIg55ijA6Nk9So4S8wD+Y8rOnaQ2Z2QH330Uc2dO9egVEDGLly4oJo1a6bbXrNmTV24cMGARM4tNDRUH3zwgYoWLarQ0NC7Hjtp0qR8SmUOt27d0tixY/XKK6/ogQceuOuxL730kjw9PfMpGfCX1NRUoyMAyARXdOzgxIkTNs9dXFxUpkwZFSpUyKBEQOYCAwMVGBioadOm2Wzv06ePduzYoW3bthmUzDk9+eSTWrFihYoXL64nn3wy0+MsFot+/PHHfExmDh4eHtqzZ498fX2NjgLYTZ06dbRmzRr5+PgYHQUwNYpOPuIXGxzB5s2b1bZtW1WqVMl6n4eIiAjFxsZq7dq1TJiFQ2nfvr2ef/55de/e3egogN0w1BLIHwxdy0fHjx+33qARMMoTTzyhgwcPaubMmdY7dz///PN68803VaFCBYPTmcuVK1f0448/qmbNmhkOF8S9tW7dWoMGDdKePXvk7++fbin0v881AwDgT1zRyUecwYGjuHHjhnbv3q0zZ86kG1/OB8ec+9e//qVmzZopJCREf/zxh+rVq6fjx48rLS2Nu6Ln0N0mejO5G86KzwNA/uCKDnCfWbdunbp166bz58+nW0CDD465s2XLFg0ZMkSStGLFCqWlpenSpUuaP3++Ro8eTdHJASZ6AwByijsDAveZPn366IUXXtCpU6eUmppq86Dk5M7ly5dVsmRJSbcLZceOHVWkSBG1bdtWhw8fNjid87tx44bREQAAToSiA9xnEhISFBoaKm9vb6OjmI6Pj48iIiKUmJiodevWqWXLlpKkixcvsgpjDqWkpOiDDz5QxYoVVaxYMR09elSSNGzYMM2ZM8fgdMDdMTsAMBZFB7jP/POf/9SmTZuMjmFK77zzjrp27aoHHnhAFSpUUPPmzSXdHtJWp04dY8M5qTFjxmjevHmaMGGC3NzcrNtr166t2bNnG5gMuDd3d3froi93+uSTTzjZBOQDFiPIRwsWLFD79u3TrRoE5Kfr16/rhRdeUJkyZVSnTh0VLFjQZv/bb79tUDJziIyMVGxsrJ5++mkVK1ZMkrR69WoVL15cTZo0MTid86levbo++eQTtWjRwmYC94EDB9S4cWNdvHjR6IhApjcLnjp1ql566SWVKlVKEjcNBvIbRcdOwsLCNHnyZOuZm1q1aumdd95RUFCQwckAW3PmzFHv3r1VqFAhlSpVShaLxbrPYrFYhwYh73h6eio6OpoVl7KgcOHCOnDggCpXrmxTdPbt26dGjRrp2rVrRkcE5OLionr16ql48eI22zdv3qyAgAAVLVqUmwYDBmDVNTv4+OOP1bdvX/3zn/9U3759JUnbtm1TmzZtNHnyZL311lsGJwT+MmTIEI0aNUqDBg2669K9yDucX8o6Pz8//fTTT6pcubLN9qVLl6pBgwYGpQJsjR07Vp9++qkmTpyop556yrq9YMGCmjdvnvz8/AxMB9y/KDp2MHbsWE2ePFkhISHWbW+//baaNGmisWPHUnTgUJKTk9WpUydKDpzC8OHD1b17d/3+++9KTU3V8uXLdfDgQX3xxRf63//+Z3Q8QJI0aNAgtWjRQi+99JKeffZZjRs3Lt2wYAD5j086dnDp0iU988wz6ba3bNlSly9fNiARkLnu3btr0aJFRscAsqR9+/b67rvvtGHDBhUtWlTDhw/X/v379d133+npp582Oh5g9cgjjygyMlJnz55VQECA9u7dazM0GED+44qOHbRr104rVqzQgAEDbLavWrVK//jHPwxKBWQsJSVFEyZM0Pfff6+6deumO+vIZFk4mqZNm+qHH34wOgZwT8WKFdP8+fO1cOFCBQUFcW8ywGAUnRyaNm2a9Ws/Pz+NGTNGmzZtUuPGjSXdnqMTHh6ufv36GRURyNCePXuscxv27t1rs4+zj/mD9xkwt86dO+vxxx9XZGRkuvllAPIPq67lUJUqVbJ0HKtYAfi7O1cPQ3olSpTIchm8cOFCHqcBADgrrujk0LFjx4yOAMBJxMXFacSIEZo7d64kae3atapYsaLBqRzXlClTrF+fP39eo0ePVqtWraxXzCMiIvT9999r2LBhBiUEADgDrugAQB7btWuXGjZsyHj9HOjYsaOefPJJm1UtJWn69OnasGGDVq5caUwwAIDDo+jYwSuvvHLX/X+exQVgTt9+++1d9x89elT9+vWj6ORAsWLFFB0drerVq9tsj4mJUf369blhKAAgUwxds4OLFy/aPL9586b27t2rS5cu2dw4DIA5dejQQRaL5a43AmUBgpwpVaqUVq1alW5hl1WrVqlUqVIGpQIAOAOKjh2sWLEi3bbU1FS98cYbqlatmgGJAOSn8uXL6+OPP1b79u0z3B8dHS1/f/98TmUOo0aNUs+ePbVp0yYFBgZKkn755RetW7dOn332mcHpAACOjBuG5hEXFxeFhoZq8uTJRkcBkMf8/f0VGRmZ6f57Xe1B5nr06KHw8HB5enpq+fLlWr58uTw9PbV161b16NHD6HgAAAfGFZ08dOTIEd26dcvoGADy2IABA5SYmJjp/urVq2vjxo35mMhcAgMD9fXXXxsdAwDgZFiMwA5CQ0Ntnqelpen06dNavXq1unfvrunTpxuUDACcX0pKilauXKn9+/dLkh5++GG1a9dOrq6uBicDADgyio4dPPnkkzbPXVxcVKZMGT311FN65ZVXVKAAF84AICdiYmLUtm1bnTx5UjVq1JAkHTx4UD4+Plq9ejXzIAEAmaLoAAAcVps2bZSWlqavv/5aJUuWlHT7JqIvvfSSXFxctHr1aoMTAgAcFUUHAOCwihYtqm3btqlOnTo223ft2qUmTZpwHx0AQKZYdc0OEhIS9PLLL6tChQoqUKCAXF1dbR4AgJxxd3fX1atX022/du2a3NzcDEgEAHAWTB6xgx49eig2NlbDhg1T+fLluTEgANjJP/7xD7322muaM2eOGjVqJOn2fXR69+6tdu3aGZwOAODIGLpmBx4eHvrpp59Uv359o6MAgKlcunRJ3bt313fffaeCBQtKkm7duqV27dpp3rx58vLyMjghAMBRcUXHDnx8fLgZIADkgeLFi2vVqlWKiYmxLi9dq1YtVa9e3eBkAABHxxUdO1i/fr0mTpyoTz75RL6+vkbHAQAAAO57FJ0cKlGihM1cnMTERN26dUtFihSxDq/404ULF/I7HgCYQseOHdWoUSMNHDjQZvuECRO0Y8cOLVmyxKBkAABHR9HJofnz52f52O7du+dhEgAwrzJlyujHH39Mt7z0nj17FBQUpISEBIOSAQAcHXN0cign5WX8+PHq3bu3ihcvbv9AAGBCmS0jXbBgQV25csWARAAAZ8F9dPLR2LFjGcYGANlQp04dLVq0KN32hQsXys/Pz4BEAABnwRWdfMQoQQDInmHDhun555/XkSNH9NRTT0mSwsLC9M033zA/BwBwVxQdAIDDevbZZ7Vy5UqNHTtWS5cuVeHChVW3bl1t2LBBTzzxhNHxAAAOjMUI8pGHh4d27dqlqlWrGh0FAAAAMDWu6AAAHF5ycrLOnDmj1NRUm+2VKlUyKBEAwNFRdAAADuvw4cN65ZVX9PPPP9tsT0tLk8ViUUpKikHJAACOjqKTQ6Ghofrggw9UtGhRbdmyRY899pgKFLj729m0aVMVLlw4nxICgPPr0aOHChQooP/9738qX768zY2aAQC4G+bo5FDBggV18uRJeXt7y9XVVadPn1bZsmWNjgUAplK0aFFFRkaqZs2aRkcBADgZrujkkK+vr6ZNm6aWLVsqLS1NERERKlGiRIbHNmvWLJ/TAYA5+Pn56dy5c0bHAAA4Ia7o5NDKlSvVu3dvnTlzRhaLJdN75DCGHABy7scff9TQoUM1duxY1alTRwULFrTZ7+npaVAyAICjo+jk0rVr1+Tp6amDBw9mOnTNy8srn1MBgDm4uLhIUrq5OSxGAAC4F4au5VKxYsW0ceNGValS5Z6LEQAAsmfjxo1GRwAAOCmu6NhBZosRnD9/XmXLluWMIwAAAJDPXIwOYAaZdcWkpCS5ubnlcxoAMJeffvpJL730kh577DH9/vvvkqQvv/xSW7duNTgZAMCRMdYqF6ZNmybp9tjx2bNnq1ixYtZ9KSkp2rJlC0uiAkAuLFu2TC+//LK6du2qqKgoJSUlSZIuX76ssWPHas2aNQYnBAA4Koau5UKVKlUkSSdOnNADDzwgV1dX6z43Nzf5+vrq/fffV2BgoFERAcCpNWjQQO+++666desmDw8P7dq1S1WrVtXOnTvVunVrxcfHGx0RAOCguKKTC8eOHZMkPfnkk1q+fHmm99EBAOTMwYMHM7wXmZeXly5dupT/gQAAToM5OnawcePGLJUcT09PHT16NB8SAYA5lCtXTjExMem2b926VVWrVjUgEQDAWVB08hGjBAEge3r16qW+ffvql19+kcVi0alTp/T111+rf//+euONN4yOBwBwYAxdAwA4rEGDBik1NVUtWrTQ9evX1axZM7m7u6t///7q06eP0fEAAA6MxQjy0Z0TaQEAWZecnKyYmBhdu3ZNfn5+NqtcStLJkydVoUIFubgwUAEAcBtXdAAADs/NzU1+fn6Z7vfz81N0dDQnkgAAVpz6ykcWi8XoCABgSgxOAAD8HUUnH/GHGAAAAMgfFJ18tHbtWlWsWNHoGAAAAIDpUXRyISoqynrTUEn68ssv1aRJE/n4+Ojxxx/XwoULbY5//PHH5e7unt8xAQAAgPsORScXgoODdeTIEUnS7Nmz9frrrysgIEBDhgzRI488ol69emnu3LkGpwQA82MOJADg71h1LRcOHz6sBx98UJL08ccfa+rUqerVq5d1/yOPPKIxY8bolVdeMSoiANwXmAMJAPg7rujkQpEiRXTu3DlJ0u+//65GjRrZ7A8MDLQZ2gYAyJmYmBh9//33+uOPPySlLzb79u1T5cqVjYgGAHBQFJ1caN26tWbOnClJeuKJJ7R06VKb/YsXL1b16tWNiAYApnD+/HkFBQXpoYceUps2bXT69GlJ0quvvqp+/fpZj/Px8ZGrq6tRMQEADsiSxvX+HDt16pSaNGmiSpUqKSAgQDNnzpS/v79q1aqlgwcPatu2bVqxYoXatGljdFQAcErdunXTmTNnNHv2bNWqVUu7du1S1apV9f333ys0NFS//fab0REBAA6KOTq5UKFCBe3cuVPjx4/Xd999p7S0NG3fvl1xcXFq0qSJwsPDFRAQYHRMAHBa69ev1/fff68HHnjAZvuDDz6oEydOGJQKAOAMKDq5VLx4cY0fP17jx483OgoAmE5iYqKKFCmSbvuFCxdYrh8AcFfM0QEAOKymTZvqiy++sD63WCxKTU3VhAkT9OSTTxqYDADg6JijAwBwWHv37lWLFi3UsGFD/fjjj2rXrp1+++03XbhwQeHh4apWrZrREQEADoqiAwBwaJcvX9b06dO1a9cuXbt2TQ0bNtRbb72l8uXLGx0NAODAKDoAAAAATIc5OgAAh7Vu3Tpt3brV+nzGjBmqX7++unTpoosXLxqYDADg6Cg6AACHNWDAAF25ckWStGfPHoWGhqpNmzY6duyYQkNDDU4HAHBkLC8NAHBYx44dk5+fnyRp2bJlevbZZzV27FhFRUVxM2YAwF1xRQcA4LDc3Nx0/fp1SdKGDRvUsmVLSVLJkiWtV3oAAMgIV3QAAA7r8ccfV2hoqJo0aaLt27dr0aJFkqRDhw7pgQceMDgdAMCRcUUHAOCwpk+frgIFCmjp0qWaOXOmKlasKElau3atnnnmGYPTAQAcGctLAwAAADAdhq4BABxWbGzsXfdXqlQpn5IAAJwNV3QAAA7LxcVFFosl0/0pKSn5mAYA4Ey4ogMAcFg7d+60eX7z5k3t3LlTkyZN0pgxYwxKBQBwBlzRAQA4ndWrV+vDDz/Upk2bjI4CAHBQrLoGAHA6NWrU0I4dO4yOAQBwYAxdAwA4rL/fFDQtLU2nT5/WyJEj9eCDDxqUCgDgDCg6AACHVbx48XSLEaSlpcnHx0cLFy40KBUAwBkwRwcA4LA2b95s89zFxUVlypRR9erVVaAA5+oAAJmj6AAAnF7btm01e/ZslS9f3ugoAAAHwWIEAACnt2XLFv3xxx9GxwAAOBCKDgAAAADToegAAAAAMB2KDgAAAADToegAAAAAMB2KDgAAAADToegAAJzev//9b5UsWdLoGAAAB8J9dAAADu3gwYP673//q/3790uSatWqpT59+qhGjRoGJwMAODKu6AAAHNayZctUu3ZtRUZGql69eqpXr56ioqJUu3ZtLVu2zOh4AAAHxhUdAIDDqlatmrp27ar333/fZvuIESP01Vdf6ciRIwYlAwA4OooOAMBhFSlSRLt371b16tVtth8+fFj16tXT9evXDUoGAHB0DF0DADis5s2b66effkq3fevWrWratKkBiQAAzqKA0QEAALjTt99+a/26Xbt2GjhwoCIjI/Xoo49KkrZt26YlS5Zo1KhRRkUEADgBhq4BAByKi0vWBhtYLBalpKTkcRoAgLOi6AAAAAAwHeboAAAAADAd5ugAABzW35eV/rvhw4fnUxIAgLNh6BoAwGE1aNDA5vnNmzd17NgxFShQQNWqVVNUVJRByQAAjo4rOgAAh7Vz5850265cuaIePXroueeeMyARAMBZcEUHAOB09uzZo2effVbHjx83OgoAwEGxGAEAwOlcvnxZly9fNjoGAMCBMXQNAOCwpk2bZvM8LS1Np0+f1pdffqnWrVsblAoA4AwYugYAcFhVqlSxee7i4qIyZcroqaee0uDBg+Xh4WFQMgCAo6PoAAAAADAd5ugAAAAAMB3m6AAAHFZiYqLGjx+vsLAwnTlzRqmpqTb7jx49alAyAICjo+gAABxWz549tXnzZr388ssqX768LBaL0ZEAAE6COToAAIdVvHhxrV69Wk2aNDE6CgDAyTBHBwDgsEqUKKGSJUsaHQMA4IQoOgAAh/XBBx9o+PDhun79utFRAABOhqFrAACH0qBBA5u5ODExMUpLS5Ovr68KFixoc2xUVFR+xwMAOAkWIwAAOJQOHToYHQEAYAJc0QEAOL1vvvlG7dq1U9GiRY2OAgBwEBQdAIDT8/T0VHR0tKpWrWp0FACAg2AxAgCA0+OcHQDg7yg6AAAAAEyHogMAAADAdCg6AAAAAEyHogMAAADAdCg6AACHMm3aNN24cUOSFBsbm6WFBipXrpzuZqIAgPsby0sDABxKgQIFdOrUKZUtW1aurq46ffq0ypYta3QsAICTKWB0AAAA7lShQgUtW7ZMbdq0UVpamk6ePGm9wvN3lSpVyud0AABnwRUdAIBD+fTTT9WnTx/dunUr02PS0tJksViUkpKSj8kAAM6EogMAcDhXr17ViRMnVLduXW3YsEGlSpXK8Lh69erlczIAgLOg6AAAHNb8+fPVuXNnubu7Gx0FAOBkWHUNAOCwRo0apWvXrqXbfunSJVWtWtWARAAAZ0HRAQA4rOPHj2c4DycpKUm///67AYkAAM6CVdcAAA7n22+/tX79/fffy8vLy/o8JSVFYWFh8vX1NSAZAMBZMEcHAOBwXFxuDziwWCzpbhhasGBB+fr6auLEifrHP/5hRDwAgBOg6AAAHFaVKlW0Y8cOlS5d2ugoAAAnwxwdAIDDOnbsWJZKTp06dRQXF5cPiQAAzoKiAwBwesePH9fNmzeNjgEAcCAUHQAAAACmQ9EBAAAAYDoUHQAAAACmQ9EBAAAAYDoUHQAAAACmQ9EBADicPn366Keffsry8Z988om8vb3zMBEAwNlww1AAgMNxcXGRxWJRtWrV9Oqrr6p79+4qV66c0bEAAE6EKzoAAIe0fv16tWnTRh999JEqVaqk9u3b63//+59SU1ONjgYAcAIUHQCAQ6pTp46mTJmiU6dO6auvvlJSUpI6dOggHx8fDRkyRDExMUZHBAA4MIauAQAcjouLi+Lj41W2bFmb7bGxsZo7d67mzZunuLg4paSkGJQQAODoKDoAAIeTWdH5U1pamjZs2KCnn346n5MBAJwFQ9cAAA6ncuXKcnV1zXS/xWKh5AAA7oorOgAAAABMhys6AAAAAEyHogMAAADAdCg6AAAAAEyHogMAAADAdCg6AAAAAEyHogMAAADAdCg6AAAAAEyHogMAAADAdP4PbTlb9AQdBtIAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_sentences, train_labels = (train_df_shuffled[\"text\"].to_numpy(),\n",
        "                                 train_df_shuffled[\"target\"].to_numpy())"
      ],
      "metadata": {
        "id": "t82s7LEaeKWO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(train_sentences), len(train_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PbJp0RgDfHzu",
        "outputId": "c995e7e8-f83d-4e52-8c08-efb1200ec50b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(7613, 7613)"
            ]
          },
          "metadata": {},
          "execution_count": 145
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_sentences = (test_df[\"text\"].to_numpy())\n",
        "len(test_sentences)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "767B2OvxieCs",
        "outputId": "ce0bbdda-ebd4-4ce9-a406-36cdc70d8e9f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3263"
            ]
          },
          "metadata": {},
          "execution_count": 150
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Training on the model with best yet results\n",
        "model_6_new_history = model_6.fit(train_sentences,\n",
        "                                  train_labels,\n",
        "                                  epochs=5,\n",
        "                                  validation_data=(val_sentences, val_labels))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OzqZ3CZHfLqc",
        "outputId": "599d070f-45eb-4e6c-9c9f-abacbe5cf1e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "238/238 [==============================] - 4s 16ms/step - loss: 0.3398 - accuracy: 0.8524 - val_loss: 0.3499 - val_accuracy: 0.8465\n",
            "Epoch 2/5\n",
            "238/238 [==============================] - 3s 12ms/step - loss: 0.3324 - accuracy: 0.8564 - val_loss: 0.3398 - val_accuracy: 0.8491\n",
            "Epoch 3/5\n",
            "238/238 [==============================] - 3s 12ms/step - loss: 0.3257 - accuracy: 0.8600 - val_loss: 0.3305 - val_accuracy: 0.8543\n",
            "Epoch 4/5\n",
            "238/238 [==============================] - 3s 12ms/step - loss: 0.3184 - accuracy: 0.8659 - val_loss: 0.3238 - val_accuracy: 0.8648\n",
            "Epoch 5/5\n",
            "238/238 [==============================] - 4s 15ms/step - loss: 0.3101 - accuracy: 0.8689 - val_loss: 0.3143 - val_accuracy: 0.8635\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make some predictions with our best yet model\n",
        "model_6_new_pred_probs = model_6.predict(val_sentences)\n",
        "# Convert model 51 pred probs to labels\n",
        "model_6_new_preds = tf.squeeze(tf.round(model_6_new_pred_probs))\n",
        "# Evaluate model 51 predictions\n",
        "model_6_new_results = calculate_results(y_true=val_labels,\n",
        "                                        y_pred=model_6_new_preds)\n",
        "model_6_new_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "684Hp6q8f0FF",
        "outputId": "150f574c-f33b-4247-dada-2aa4e1fa8561"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 0s 9ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 86.3517060367454,\n",
              " 'precision': 0.8655202814561257,\n",
              " 'recall': 0.863517060367454,\n",
              " 'f1': 0.8622819689338196}"
            ]
          },
          "metadata": {},
          "execution_count": 154
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Making some predictions on our test data\n",
        "model_6_new_pred_probs_test = model_6.predict(test_sentences)\n",
        "# Convert model 6 pred probs to labels\n",
        "model_6_new_preds_test = tf.squeeze(tf.round(model_6_new_pred_probs_test))\n",
        "model_6_new_preds_test[:20]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uWwD5CcskWpj",
        "outputId": "12534bc2-6c97-42eb-f905-fdb6f5544b95"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "102/102 [==============================] - 1s 9ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(20,), dtype=float32, numpy=\n",
              "array([1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,\n",
              "       0., 0., 0.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 156
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from helper_functions import make_confusion_matrix"
      ],
      "metadata": {
        "id": "vC3nzKrVgvi9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a confusion matrix with the best yet performing model's predictions\n",
        "make_confusion_matrix(y_true=val_labels,\n",
        "                      y_pred=model_6_new_preds,\n",
        "                      figsize=(10,10))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 807
        },
        "id": "RR248RcmhEud",
        "outputId": "3e3b33d6-cb8c-44bd-c7c4-0ef75af6ba7d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x1000 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxcAAAMWCAYAAACDfHuZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABb60lEQVR4nO3de5xN9f7H8ffaY2YPY/aMcZkxGeNWmNxKpancck/FoVOimhxxEl0oSeV+Skf3i+iUUD9OdzoukUuoTImIkBCZYhCZYZj7/v0xZtduyGx9x17bfj3nsR5Za333Wp81PdJ85vP5fpfldrvdAgAAAIC/yOHvAAAAAACcG0guAAAAABhBcgEAAADACJILAAAAAEaQXAAAAAAwguQCAAAAgBEkFwAAAACMILkAAAAAYEQ5fwcAAAAA+Et2drZyc3P9HUYJYWFhCg8P93cYPiO5AAAAQFDKzs5W+cjKUv4xf4dSQlxcnHbu3BlwCQbJBQAAAIJSbm6ulH9MzqQUKSTM3+H8piBX6ZtnKDc3l+QCAAAACCghYbJslFy4/R3AX0ByAQAAgOBmOYo2u7BTLD4K3MgBAAAA2ArJBQAAAAAjaIsCAABAcLMkWZa/o/iNjULxFZULAAAAAEaQXAAAAAAwgrYoAAAABDdWizImcCMHAAAAYCskFwAAAACMoC0KAAAAwc2ybLZalI1i8RGVCwAAAABGkFwAAAAAMIK2KAAAAAQ3VosyJnAjBwAAAGArJBcAAAAAjKAtCgAAAMGN1aKMoXIBAAAAwAiSCwAAAABG0BYFAACAIGez1aIC+Pf/gRs5AAAAAFshuQAAAABgBG1RAAAACG6sFmUMlQsAAAAARpBcAAAAADCCtigAAAAEN8tmq0XZKRYfBW7kAAAAAGyF5AIAAACAEbRFAQAAILixWpQxVC4AAAAAGEFyAQAAAMAI2qIAAAAQ3FgtypjAjRwAAACArZBcAAAAADCCtigAAAAEN1aLMobKBQAAAAAjSC4AAAAAGEFbFAAAAIIbq0UZE7iRAwAAALAVkgsAAAAARtAWBQAAgOBmWfZqRWK1KAAAAADBjuQCAAAAgBG0RQEAACC4OayizS7sFIuPqFwAAAAAMILkAgAAAIARtEUBAAAguPESPWMCN3IAAAAAtkJyAQAAAMAI2qIAAAAQ3CzLXi+us1MsPqJyAQAAAMAIkgsAQW/btm3q2LGjoqKiZFmW5syZY/T6u3btkmVZmj59utHrBrI2bdqoTZs2/g4DAGAYyQUAW9ixY4f++c9/qk6dOgoPD5fL5dKVV16p559/XsePHy/Te6ekpGjjxo167LHH9Oabb+qSSy4p0/udTbfffrssy5LL5Trp93Hbtm2yLEuWZempp57y+fp79uzRmDFjtH79egPRAoCfFK8WZactQDHnAoDfzZ8/X3//+9/ldDp12223qVGjRsrNzdVnn32mYcOGadOmTfrPf/5TJvc+fvy4UlNT9cgjj2jw4MFlco/ExEQdP35coaGhZXL90ylXrpyOHTumuXPn6sYbb/Q6N3PmTIWHhys7O/uMrr1nzx6NHTtWtWrVUrNmzUr9uY8//viM7gcAsDeSCwB+tXPnTvXq1UuJiYlatmyZqlev7jk3aNAgbd++XfPnzy+z+x84cECSFB0dXWb3sCxL4eHhZXb903E6nbryyiv13//+t0RyMWvWLHXt2lXvv//+WYnl2LFjqlChgsLCws7K/QAAZ1fg1lwAnBMmTpyoo0ePaurUqV6JRbF69erp3nvv9ezn5+dr/Pjxqlu3rpxOp2rVqqWHH35YOTk5Xp+rVauWrr32Wn322We67LLLFB4erjp16uiNN97wjBkzZowSExMlScOGDZNlWapVq5akonai4j//3pgxY2T9YRWPxYsX66qrrlJ0dLQqVqyo+vXr6+GHH/acP9Wci2XLlqlly5aKiIhQdHS0unXrpi1btpz0ftu3b9ftt9+u6OhoRUVFqW/fvjp27Nipv7F/0Lt3b3300Uc6fPiw59hXX32lbdu2qXfv3iXGHzp0SA888IAaN26sihUryuVyqUuXLvrmm288Y5YvX65LL71UktS3b19Pe1Xxc7Zp00aNGjXS2rVr1apVK1WoUMHzffnjnIuUlBSFh4eXeP5OnTqpUqVK2rNnT6mfFQB8VrxalJ22AEVyAcCv5s6dqzp16uiKK64o1fg77rhDo0aN0sUXX6xnn31WrVu31oQJE9SrV68SY7dv364bbrhBHTp00NNPP61KlSrp9ttv16ZNmyRJPXr00LPPPitJuvnmm/Xmm2/queee8yn+TZs26dprr1VOTo7GjRunp59+Wtdff70+//zzP/3ckiVL1KlTJ+3fv19jxozR0KFDtWrVKl155ZXatWtXifE33nijjhw5ogkTJujGG2/U9OnTNXbs2FLH2aNHD1mWpQ8++MBzbNasWWrQoIEuvvjiEuN/+OEHzZkzR9dee62eeeYZDRs2TBs3blTr1q09P+g3bNhQ48aNkyQNGDBAb775pt588021atXKc52DBw+qS5cuatasmZ577jm1bdv2pPE9//zzqlq1qlJSUlRQUCBJeuWVV/Txxx/rxRdfVHx8fKmfFQDgP7RFAfCbzMxM/fzzz+rWrVupxn/zzTeaMWOG7rjjDr366quSpLvuukvVqlXTU089pU8++cTrh9etW7dq5cqVatmypaSiH9ATEhI0bdo0PfXUU2rSpIlcLpeGDBmiiy++WLfccovPz7B48WLl5ubqo48+UpUqVUr9uWHDhikmJkapqamKiYmRJHXv3l0XXXSRRo8erRkzZniNv+iiizR16lTP/sGDBzV16lT9+9//LtX9IiMjde2112rWrFn6xz/+ocLCQr311lsaOHDgScc3btxY33//vRyO334Hdeutt6pBgwaaOnWqRo4cqdjYWHXp0kWjRo1ScnLySb9/6enpmjJliv75z3/+aXzR0dGaOnWqOnXqpCeeeEK9e/fWAw88oO7du5/RvxcAgH9QuQDgN5mZmZKKfvAtjQULFkiShg4d6nX8/vvvl6QSczOSkpI8iYUkVa1aVfXr19cPP/xwxjH/UfFcjQ8//FCFhYWl+szevXu1fv163X777Z7EQpKaNGmiDh06eJ7z9+68806v/ZYtW+rgwYOe72Fp9O7dW8uXL1d6erqWLVum9PT0k7ZESUXzNIoTi4KCAh08eNDT8vX111+X+p5Op1N9+/Yt1diOHTvqn//8p8aNG6cePXooPDxcr7zySqnvBQBnzN8rQ51Dq0UFbuQAAp7L5ZIkHTlypFTjf/zxRzkcDtWrV8/reFxcnKKjo/Xjjz96Ha9Zs2aJa1SqVEm//vrrGUZc0k033aQrr7xSd9xxh2JjY9WrVy+98847f5poFMdZv379EucaNmyoX375RVlZWV7H//gslSpVkiSfnuWaa65RZGSk3n77bc2cOVOXXnppie9lscLCQj377LM6//zz5XQ6VaVKFVWtWlUbNmxQRkZGqe953nnn+TR5+6mnnlJMTIzWr1+vF154QdWqVSv1ZwEA/kdyAcBvXC6X4uPj9e233/r0uT9OqD6VkJCQkx53u91nfI/i+QDFypcvr5UrV2rJkiW69dZbtWHDBt10003q0KFDibF/xV95lmJOp1M9evTQjBkzNHv27FNWLSTp8ccf19ChQ9WqVSv93//9nxYtWqTFixfrwgsvLHWFRir6/vhi3bp12r9/vyRp48aNPn0WAOB/JBcA/Oraa6/Vjh07lJqaetqxiYmJKiws1LZt27yO79u3T4cPH/as/GRCpUqVvFZWKvbH6ogkORwOtWvXTs8884w2b96sxx57TMuWLdMnn3xy0msXx7l169YS57777jtVqVJFERERf+0BTqF3795at26djhw5ctJJ8MXee+89tW3bVlOnTlWvXr3UsWNHtW/fvsT3pLSJXmlkZWWpb9++SkpK0oABAzRx4kR99dVXxq4PAKfk75WhWC0KAMx48MEHFRERoTvuuEP79u0rcX7Hjh16/vnnJRW19UgqsaLTM888I0nq2rWrsbjq1q2rjIwMbdiwwXNs7969mj17tte4Q4cOlfhs8cvk/rg8brHq1aurWbNmmjFjhtcP699++60+/vhjz3OWhbZt22r8+PF66aWXFBcXd8pxISEhJaoi7777rn7++WevY8VJ0MkSMV8NHz5cu3fv1owZM/TMM8+oVq1aSklJOeX3EQBgP6wWBcCv6tatq1mzZummm25Sw4YNvd7QvWrVKr377ru6/fbbJUlNmzZVSkqK/vOf/+jw4cNq3bq1Vq9erRkzZqh79+6nXOb0TPTq1UvDhw/X3/72N91zzz06duyYJk+erAsuuMBrQvO4ceO0cuVKde3aVYmJidq/f79efvll1ahRQ1ddddUpr//kk0+qS5cuSk5OVr9+/XT8+HG9+OKLioqK0pgxY4w9xx85HA49+uijpx137bXXaty4cerbt6+uuOIKbdy4UTNnzlSdOnW8xtWtW1fR0dGaMmWKIiMjFRERoRYtWqh27do+xbVs2TK9/PLLGj16tGdp3GnTpqlNmzYaOXKkJk6c6NP1AAD+QXIBwO+uv/56bdiwQU8++aQ+/PBDTZ48WU6nU02aNNHTTz+t/v37e8a+9tprqlOnjqZPn67Zs2crLi5OI0aM0OjRo43GVLlyZc2ePVtDhw7Vgw8+qNq1a2vChAnatm2bV3Jx/fXXa9euXXr99df1yy+/qEqVKmrdurXGjh2rqKioU16/ffv2WrhwoUaPHq1Ro0YpNDRUrVu31r///W+ffzAvCw8//LCysrI0a9Ysvf3227r44os1f/58PfTQQ17jQkNDNWPGDI0YMUJ33nmn8vPzNW3aNJ+e4ciRI/rHP/6hiy66SI888ojneMuWLXXvvffq6aefVo8ePXT55Zcbez4A8GK3FZrsFIuPLLcvswEBAACAc0RmZqaioqLkbPeYrHLh/g7Hw52frZyljygjI8OzsmKgCNy0CAAAAICt0BYFAACA4Ga3FZrsFIuPqFwAAAAAMILkAgAAAIARtEUBAAAgyNlstagA/v1/4EYOAAAAwFYCunJRWFioPXv2KDIyUlYAT3wBAAA4V7ndbh05ckTx8fFyOPi99rkuoJOLPXv2KCEhwd9hAAAA4DTS0tJUo0YNf4dxcqwWZUxAJxeRkZGSpLCkFFkhYX6OBgDM2L38KX+HAADGHMnMVL3aCZ6f23BuC+jkorgVygoJI7kAcM4ItLexAkBp0MIeHAI6uQAAAAD+Msuy12pRAZyI2ei7CAAAACCQkVwAAAAAMIK2KAAAAAQ3y2Yv0bNTLD4K3MgBAAAA2ArJBQAAAAAjaIsCAABAcOMlesZQuQAAAABgBMkFAAAAACNoiwIAAEBwY7UoYwI3cgAAAAC2QnIBAAAAwAjaogAAABDcWC3KGCoXAAAAAIwguQAAAABgBMkFAAAAglvxalF22nwwefJkNWnSRC6XSy6XS8nJyfroo48859u0aSPLsry2O++80+sau3fvVteuXVWhQgVVq1ZNw4YNU35+vs/fSuZcAAAAAAGsRo0aeuKJJ3T++efL7XZrxowZ6tatm9atW6cLL7xQktS/f3+NGzfO85kKFSp4/lxQUKCuXbsqLi5Oq1at0t69e3XbbbcpNDRUjz/+uE+xkFwAAAAAAey6667z2n/sscc0efJkffHFF57kokKFCoqLizvp5z/++GNt3rxZS5YsUWxsrJo1a6bx48dr+PDhGjNmjMLCwkodC21RAAAACG7Fq0XZaTtDBQUFeuutt5SVlaXk5GTP8ZkzZ6pKlSpq1KiRRowYoWPHjnnOpaamqnHjxoqNjfUc69SpkzIzM7Vp0yaf7k/lAgAAALChzMxMr32n0ymn03nSsRs3blRycrKys7NVsWJFzZ49W0lJSZKk3r17KzExUfHx8dqwYYOGDx+urVu36oMPPpAkpaeneyUWkjz76enpPsVMcgEAAADYUEJCgtf+6NGjNWbMmJOOrV+/vtavX6+MjAy99957SklJ0YoVK5SUlKQBAwZ4xjVu3FjVq1dXu3bttGPHDtWtW9dozCQXAAAACGrFKyjZxolY0tLS5HK5PIdPVbWQpLCwMNWrV0+S1Lx5c3311Vd6/vnn9corr5QY26JFC0nS9u3bVbduXcXFxWn16tVeY/bt2ydJp5yncSrMuQAAAABsqHhp2eLtz5KLPyosLFROTs5Jz61fv16SVL16dUlScnKyNm7cqP3793vGLF68WC6Xy9NaVVpULgAAAIAANmLECHXp0kU1a9bUkSNHNGvWLC1fvlyLFi3Sjh07NGvWLF1zzTWqXLmyNmzYoCFDhqhVq1Zq0qSJJKljx45KSkrSrbfeqokTJyo9PV2PPvqoBg0a5FNCI5FcAAAAIMjZtS2qtPbv36/bbrtNe/fuVVRUlJo0aaJFixapQ4cOSktL05IlS/Tcc88pKytLCQkJ6tmzpx599FHP50NCQjRv3jwNHDhQycnJioiIUEpKitd7MUqL5AIAAAAIYFOnTj3luYSEBK1YseK010hMTNSCBQv+cizMuQAAAABgBJULAAAABDfrxGYXdorFR1QuAAAAABhBcgEAAADACNqiAAAAENQCfbUoO6FyAQAAAMAIkgsAAAAARtAWBQAAgKBGW5Q5VC4AAAAAGEFyAQAAAMAI2qIAAAAQ1GiLMofKBQAAAAAjSC4AAAAAGEFbFAAAAIIabVHmULkAAAAAYATJBQAAAAAjaIsCAABAcLNObHZhp1h8ROUCAAAAgBEkFwAAAACMoC0KAAAAQY3VosyhcgEAAADACJILAAAAAEbQFgUAAICgZlmyWVuUvwM4c1QuAAAAABhBcgEAAADACNqiAAAAENQs2Wy1qADui6JyAQAAAMAIkgsAAAAARtAWBQAAgKDGS/TMoXIBAAAAwAiSCwAAAABG0BYFAACA4GbJXgs02SkWH1G5AAAAAGAEyQUAAAAAI2iLAgAAQHCz2WpRbhvF4isqFwAAAACMILkAAAAAYARtUQAAAAhqdnuJnp1i8RWVCwAAAABGkFwAAAAAMIK2KAAAAAQ12qLMoXIBAAAAwAiSCwAAAABG0BYFAACA4Gad2OzCTrH4iMoFAAAAACNILgAAAAAYQVsUAAAAghqrRZlD5QIAAACAESQXAAAAAIygLQoAAABBjbYoc6hcAAAAADCC5AIAAACAEbRFAQAAIKjRFmUOlQsAAAAARpBcAAAAADCCtigAAAAENdqizKFyAQAAAMAIkgsAAAAARtAWBQAAgOBmndjswk6x+IjKBQAAAAAjSC4AAAAAGEFbFAAAAIIaq0WZQ+UCAAAAgBEkFwAAAACMoC0KAAAAQY22KHOoXAAAAAAwguQCAAAAgBG0RQEAACCo0RZlDpULAAAAAEaQXAAAAAAwgrYoAAAABDfrxGYXdorFR1QuAAAAABhBcgEAAADACNqiAAAAENRYLcocKhcAAAAAjCC5AAAAAGAEbVEAAAAIarRFmUPlAgAAAIARJBcAAAAAjKAtCgAAAEHNks3aogL4LXpULgAAAAAYQXIBAAAAwAjaogAAABDUWC3KHCoXAAAAAIwguQAAAABgBG1RAAAACG7Wic0u7BSLj6hcAAAAADCC5AIAAACAESQXAAAAAIxgzgUAAACCGkvRmkPlAgAAAIARJBcAAAAAjKAtCgAAAEGNtihzqFwAAAAAMILkAgAAAIARJBcAAAAIapZlv80XkydPVpMmTeRyueRyuZScnKyPPvrIcz47O1uDBg1S5cqVVbFiRfXs2VP79u3zusbu3bvVtWtXVahQQdWqVdOwYcOUn5/v8/eS5AIAAAAIYDVq1NATTzyhtWvXas2aNbr66qvVrVs3bdq0SZI0ZMgQzZ07V++++65WrFihPXv2qEePHp7PFxQUqGvXrsrNzdWqVas0Y8YMTZ8+XaNGjfI5FsvtdruNPdlZlpmZqaioKDkb95cVEubvcADAiF+/esnfIQCAMZmZmYqtHKWMjAy5XC5/h+Ol+GfJ2oPfk8NZwd/heBTmHNPOl274S9+zmJgYPfnkk7rhhhtUtWpVzZo1SzfccIMk6bvvvlPDhg2Vmpqqyy+/XB999JGuvfZa7dmzR7GxsZKkKVOmaPjw4Tpw4IDCwkr/czaVCwAAAAS1olYky0bbmT9LQUGB3nrrLWVlZSk5OVlr165VXl6e2rdv7xnToEED1axZU6mpqZKk1NRUNW7c2JNYSFKnTp2UmZnpqX6UFkvRAgAAADaUmZnpte90OuV0Ok86duPGjUpOTlZ2drYqVqyo2bNnKykpSevXr1dYWJiio6O9xsfGxio9PV2SlJ6e7pVYFJ8vPucLKhcAAACADSUkJCgqKsqzTZgw4ZRj69evr/Xr1+vLL7/UwIEDlZKSos2bN5/FaItQuQAAAEBwO4MVmsrUiVjS0tK85lycqmohSWFhYapXr54kqXnz5vrqq6/0/PPP66abblJubq4OHz7sVb3Yt2+f4uLiJElxcXFavXq11/WKV5MqHlNaVC4AAAAAGypeWrZ4+7Pk4o8KCwuVk5Oj5s2bKzQ0VEuXLvWc27p1q3bv3q3k5GRJUnJysjZu3Kj9+/d7xixevFgul0tJSUk+xUzlAgAAAAhgI0aMUJcuXVSzZk0dOXJEs2bN0vLly7Vo0SJFRUWpX79+Gjp0qGJiYuRyuXT33XcrOTlZl19+uSSpY8eOSkpK0q233qqJEycqPT1djz76qAYNGuRTQiORXAAAACDIFa/SZBe+xrJ//37ddttt2rt3r6KiotSkSRMtWrRIHTp0kCQ9++yzcjgc6tmzp3JyctSpUye9/PLLns+HhIRo3rx5GjhwoJKTkxUREaGUlBSNGzfO59hJLgAAAIAANnXq1D89Hx4erkmTJmnSpEmnHJOYmKgFCxb85ViYcwEAAADACCoXAAAACGqWzVaLslMsvqJyAQAAAMAIkgsAAAAARtAWhaBwzy1X64pmdXTh+fGqWilS4c5y2nfwiD5du03PzliqTdv3lPhMJVcFDftHR13ftqlqxEUr42i2Pv96u554daE2fP/zn97vqub1NOjmNrq8aR1VcpXXoYxj2vD9T3r13c80f8XGsnpMAJAkPf/sM1q16jNt+najDuzfr+zsbMXGxally9Yacv8wNWrc2Gv8vLn/05wP3tf6dV8rPX2vMjIyVKlSJV3c/BINuPMuXdP1Wj89CXB2OByWHA779CK5bRSLryy32+32dxBnKjMzU1FRUXI27i8rJMzf4cDG0pY9oYjyYfp22x7t2X9YktSwbnVdUCtWuXn56nX/a/ro02894+OquLT09SGqk1BVew9kaM23uxRb2aVLGiUqL79APe99RUu/+O6k93rkn9fo0TuvUXZOnlLX/6ADvx5RfLVoNWuQoHcXrdVd42adjUdGAPv1q5f8HQICXI24KsrKylKjxk0Uf955kqQtmzdp2/ffKzQ0VG+9+4FXwnDzTTfow9kfKCnpQiXUrKmKkZH6cdcufbX6S0nSsOEjNO5fj/vlWRD4MjMzFVs5ShkZGV5vm7aD4p8lLxj6gUKcEf4Ox6MgJ0vfP9PDlt+z0yG5QFBIblpHX2/ZrZzcfK/jA/7eUs8/fJPSf8lUvc6PqqCgUJL03nP/VNfWjbXws03qM2yqjmXnSpKua9NE/33qDh3KzFLStWN09FiO1/Vuua6FXh13q1Zv2Knew6bq5xOJjCSVDw9V7fOqaPOOvWX7sAh4JBf4q1Z9/rkubt5c4eHhXsdfmfyy7rtnkGJjY7V9108qV66ogWH9unVKqFlTlStX9hq/+ssv1bVze2VlZWn12m9KVDyA0iC58F0gJxfMuUBQSP3mhxKJhST9591PtWP3AcVVcalhnThJUo3YaHVt3Vh5eQW69/G3PImFJM1dvkHvLlqrqpUildIt2eta4c5QTRjyN2UePa4bh/7HK7GQpOPZeSQWAM6KK668skRiIUn/HHiX6tStq3379mnL5s2e480uuqhEYiFJl7VooRv+fpPcbrdWLP+kTGMG/Kl4tSg7bYGK5AJBLy+/QJKUm1f0z2YNEyRJu/Yc1O69v5YYv2LN95Kka9t4/wav+9VNVaVSRX2wZJ32HTxSliEDwBkLDQ2VJIWFla7i7+t4AMGNCd0Iajd3vVQX1KqmbT/u1/bd+yVJEeFOSdLhzGMn/cyhw1mSpMYX1PA63vqyCyRJX3yzU1EVy6vXNZcqqV51Zefk6YtvftD/PtngabsCAH+Y9X9v6vutW1Xv/PNV7/zzTzv+240b9d67bys0NFTt2nc4CxECCHQkFwgqQ25rp4Z1qyuifJjq147ThfXitWf/YaWMmKbCwqLpRwd+PSpJSqgec9Jr1DqviiSpcnSEIsqHKet4UdtUwzrVJUlVK1XUug8eVfWqUZ7P3HPL1dr4/c/qcc9k/bTvcFk9HgB4eebpJ7Vl0yZlHcvS1u+2aPOmTaoeH68Zb/5XISEhJcbPnzdXcz54X3l5eUpL260vUlcpNDRUL095VXXq1vXDEwBnh2VZsmzUi2SnWHxFcoGg0v6Khrq6RQPP/o97DqrfyDe0bkua59iaTbuUnZOnuCoudbiioRav2uJ1jVuua+H5c2REuCe5iI4sL0kaNfBaff/jPt3y4FRt+P5nNagdp+cfvkkXJ9XUf5+6Qy1vfaosHxEAPJZ8vEifLFvq2a+ZmKip097Qxc2bn3T8xg3f6P/enOHZL1++vJ565nn1vuXWMo8VwLmBORcIKl3vfEnlLxqsuJbD1P4fz2r77gNaMnWIHuzXyTMm82i2/vPup5KkV8fdquvbNpGrYrjOT6ymN5/oqwa14zxji6sdkuRwFP3nVFBYqO6DX9aq9T/o6LEcrdn0o7oPfllHj+Xokka1vJIbAChLCxYt0fE8t/Ye+FWLP1mpevXOV4erW+vfEx476fiHHn5Ux/Pc+vXIca1Zt1G3pfTVoIED9Pce3ZSbm3vSzwDA75FcIChlHD2uz9ftUPe7X9bazbs1+q6uap5U03N+5Av/0weLv1ZsZZfefmaA9n36lDbMGaXr2jbRA0++5xl3+Mhxz5+Ll6X9ZPXWEq1PB349qoUn3qPRsnm9MnwyACgpOjpaV13VUnPmLtDFFzfX2NEjtearr045Pjw8XBc2aqTnXpykgYPu1oL58/TypBfPYsTA2eXvlaFYLQo4R+TnF+r9RWvlcDh0TevfVn/KzctXnwdfV/t/PKt/v7ZIU9//XGNfnqfmf39cS74oapPavnu/cvN+W942be8hSdKPew6d9F7Fx6vGRJbV4wDAnwoNDVXPG4uWll0wf26pPtO7T1FL1Lz/fViWoQE4RzDnAkHvlxOrP1WpVLHEuc/X7dDn63Z4Het97WWSpJVrtnkdX781Tddf3VSVXBVOep9KUUXHs47nnPQ8AJwNVSoXLUrxy4EDpRtf5cT4X0o3HkBws0XlYtKkSapVq5bCw8PVokULrV692t8hIYgUtyntTPulVOPvvLGVJGnaB6u8js9fvlGSdHnT2ipXzvs/LcuydEWzopVW1n+XJgDwl09XrpAk1S7l6k/F4+vUYbUonLuKV4uy0xao/J5cvP322xo6dKhGjx6tr7/+Wk2bNlWnTp20f/9+f4eGc0Ry0zrqcEXDEv+hlivn0MBerdW762U6djxX73281nMuIa6Sqv6hkhHuDNWkkTfr0sa19MaHX2jNph+9zm/4/mctSd2ixPjKGnPXdV7nHurfSQ3qxGnfwUx9uPQbw08IAL9Z9fnn+njRQhUWer9XJy8vTy+/9KJmzXxT5cuX1w1/v0mSdODAAb3+2qs6dqzku32WLlmsR0Y8KEm6NaVv2QcPIOD5vS3qmWeeUf/+/dW3b9FfWlOmTNH8+fP1+uuv66GHHvJzdDgX1K1ZVa+Ou1UHfj2idZvTdCgjS5WjK6rR+fGqXjVKx7NzNWD0m16TsFtfeoFeHtlbX2/ZrbS9hxQeHqbkpnVUOTpCH3++Wfc8/tZJ7zVw7Ewtn3G/7u/bQde1baJN2/eofu04JdWtrmPHc9X3kRk6ls2KKwDKzo7t2zTgjr6qUqWKLrq4uWJiKuvgwV/07bcblb53r8LDw/WfqdOVkJAgSTqWlaVBAwdo2P336aKLm+u8GjV0LCtL27Z9r63ffSdJuvveIfpbj57+fCwAAcKvyUVubq7Wrl2rESNGeI45HA61b99eqampJcbn5OQoJ+e3fvXMzMyzEicC26drt+vfry1Sy+b11PiC81Q5OkK5eQX6cc9BzV6yTpP+u1w//KElat2WNM1euk6XNa6lJhecp5y8fG3atkdv/O8LvfHhF6e810/7Duvym/+tRwZ00TWtG6tr68b6NfOY3lm4Rv9+bZE279hb1o8LIMi1bNVaDz70sD5duUIbN27QwV9+UVhYmBJr1dLfetygQYPvUd16v61aV7VaNT32xER9umK5Nm/epK/XrlFhYaHiqlfX32/qpTv6/1OtWrfx3wMBZ4HdWpHsFIuvLLfb7T79sLKxZ88enXfeeVq1apWSk5M9xx988EGtWLFCX375pdf4MWPGaOzYsSWu42zcX1ZIWJnHCwBnw69fveTvEADAmMzMTMVWjlJGRoZcLpe/w/GSmZmpqKgoXTj8Q4U4I/wdjkdBTpY2/bubLb9np+P3ORe+GDFihDIyMjxbWhoTYwEAAAC78GtbVJUqVRQSEqJ9+/Z5Hd+3b5/i4uJKjHc6nXI6nWcrPAAAAAQBu724zk6x+MqvlYuwsDA1b95cS5cu9RwrLCzU0qVLvdqkAAAAANif31eLGjp0qFJSUnTJJZfosssu03PPPaesrCzP6lEAAAAAAoPfk4ubbrpJBw4c0KhRo5Senq5mzZpp4cKFio2N9XdoAAAACAKWbLZalOwTi6/8nlxI0uDBgzV48GB/hwEAAADgLwio1aIAAAAA2JctKhcAAACAv7BalDlULgAAAAAYQXIBAAAAwAjaogAAABDULMtmq0XZKBZfUbkAAAAAYATJBQAAAAAjaIsCAABAUGO1KHOoXAAAAAAwguQCAAAAgBG0RQEAACCosVqUOVQuAAAAABhBcgEAAADACNqiAAAAENRYLcocKhcAAAAAjCC5AAAAAGAEbVEAAAAIaqwWZQ6VCwAAAABGkFwAAAAAMIK2KAAAAAQ3m60WJTvF4iMqFwAAAACMILkAAAAAYARtUQAAAAhqrBZlDpULAAAAAEaQXAAAAAAwgrYoAAAABDXLZqtF2SkWX1G5AAAAAGAEyQUAAAAAI2iLAgAAQFBjtShzqFwAAAAAMILkAgAAAIARtEUBAAAgqLFalDlULgAAAAAYQXIBAAAAwAjaogAAABDUWC3KHCoXAAAAAIwguQAAAABgBG1RAAAACGq0RZlD5QIAAACAESQXAAAAAIygLQoAAABBjZfomUPlAgAAAIARJBcAAAAAjKAtCgAAAEGN1aLMoXIBAAAAwAiSCwAAAABG0BYFAACAoMZqUeZQuQAAAABgBMkFAAAAACNoiwIAAEBQY7Uoc6hcAAAAADCC5AIAAACAEbRFAQAAIKhZstcKTTYKxWdULgAAAAAYQXIBAAAAwAjaogAAABDUHJYlh436ouwUi6+oXAAAAAAwguQCAAAAgBG0RQEAACCoWZbNVouyUSy+onIBAAAAwAiSCwAAAABG0BYFAACAoGZZliwb9SLZKRZfUbkAAAAAYATJBQAAAAAjaIsCAABAUHNYRZtd2CkWX1G5AAAAAGAEyQUAAAAAI2iLAgAAQHCzbLZCk41C8RWVCwAAACCATZgwQZdeeqkiIyNVrVo1de/eXVu3bvUa06ZNG8+Su8XbnXfe6TVm9+7d6tq1qypUqKBq1app2LBhys/P9ykWKhcAAABAAFuxYoUGDRqkSy+9VPn5+Xr44YfVsWNHbd68WREREZ5x/fv317hx4zz7FSpU8Py5oKBAXbt2VVxcnFatWqW9e/fqtttuU2hoqB5//PFSx0JyAQAAgKBmWUWbXfgay8KFC732p0+frmrVqmnt2rVq1aqV53iFChUUFxd30mt8/PHH2rx5s5YsWaLY2Fg1a9ZM48eP1/DhwzVmzBiFhYWVKhbaogAAAIBzSEZGhiQpJibG6/jMmTNVpUoVNWrUSCNGjNCxY8c851JTU9W4cWPFxsZ6jnXq1EmZmZnatGlTqe9N5QIAAACwoczMTK99p9Mpp9P5p58pLCzUfffdpyuvvFKNGjXyHO/du7cSExMVHx+vDRs2aPjw4dq6das++OADSVJ6erpXYiHJs5+enl7qmEkuAAAAENSsE192URxLQkKC1/HRo0drzJgxf/rZQYMG6dtvv9Vnn33mdXzAgAGePzdu3FjVq1dXu3bttGPHDtWtW9dM4CK5AAAAAGwpLS1NLpfLs3+6qsXgwYM1b948rVy5UjVq1PjTsS1atJAkbd++XXXr1lVcXJxWr17tNWbfvn2SdMp5GifDnAsAAADAhlwul9d2quTC7XZr8ODBmj17tpYtW6batWuf9trr16+XJFWvXl2SlJycrI0bN2r//v2eMYsXL5bL5VJSUlKpY6ZyAQAAgKDmsIo2u/A1lkGDBmnWrFn68MMPFRkZ6ZkjERUVpfLly2vHjh2aNWuWrrnmGlWuXFkbNmzQkCFD1KpVKzVp0kSS1LFjRyUlJenWW2/VxIkTlZ6erkcffVSDBg06bcXEK3bfQgcAAABgJ5MnT1ZGRobatGmj6tWre7a3335bkhQWFqYlS5aoY8eOatCgge6//3717NlTc+fO9VwjJCRE8+bNU0hIiJKTk3XLLbfotttu83ovRmlQuQAAAAACmNvt/tPzCQkJWrFixWmvk5iYqAULFvylWEguAAAAENQsy5Jlo7fo2SkWX9EWBQAAAMAIkgsAAAAARtAWBQAAgKBmWUWbXdgpFl9RuQAAAABgBMkFAAAAACNoiwIAAEBQc1iWHDbqRbJTLL6icgEAAADACJILAAAAAEbQFgUAAICgxmpR5lC5AAAAAGAEyQUAAAAAI2iLAgAAQFCzLEuWjXqR7BSLr6hcAAAAADCC5AIAAACAEbRFAQAAIKixWpQ5VC4AAAAAGEFyAQAAAMAI2qIAAAAQ1ByWJYeNepHsFIuvqFwAAAAAMILkAgAAAIARtEUBAAAgqFknNruwUyy+onIBAAAAwAiSCwAAAABG0BYFAACAoGZZliwbrdBkp1h8ReUCAAAAgBEkFwAAAACMoC0KAAAAQc1hFW12YadYfEXlAgAAAIARJBcAAAAAjKAtCgAAAEGN1aLMoXIBAAAAwAiSCwAAAABG0BYFAACAoBfAnUi2QuUCAAAAgBEkFwAAAACMoC0KAAAAQY3VoswpVXLxv//9r9QXvP766884GAAAAACBq1TJRffu3Ut1McuyVFBQ8FfiAQAAABCgSpVcFBYWlnUcAAAAgF84rKLNLuwUi6/+0oTu7OxsU3EAAAAACHA+JxcFBQUaP368zjvvPFWsWFE//PCDJGnkyJGaOnWq8QABAAAABAafk4vHHntM06dP18SJExUWFuY53qhRI7322mtGgwMAAADKWvFqUXbaApXPycUbb7yh//znP+rTp49CQkI8x5s2barvvvvOaHAAAAAAAofPycXPP/+sevXqlTheWFiovLw8I0EBAAAACDw+JxdJSUn69NNPSxx/7733dNFFFxkJCgAAADhbLBtugcrnN3SPGjVKKSkp+vnnn1VYWKgPPvhAW7du1RtvvKF58+aVRYwAAAAAAoDPlYtu3bpp7ty5WrJkiSIiIjRq1Cht2bJFc+fOVYcOHcoiRgAAAAABwOfKhSS1bNlSixcvNh0LAAAAcNY5LEsOG63QZKdYfHVGyYUkrVmzRlu2bJFUNA+jefPmxoICAAAAEHh8Ti5++ukn3Xzzzfr8888VHR0tSTp8+LCuuOIKvfXWW6pRo4bpGAEAAAAEAJ/nXNxxxx3Ky8vTli1bdOjQIR06dEhbtmxRYWGh7rjjjrKIEQAAACgzlmW/LVD5XLlYsWKFVq1apfr163uO1a9fXy+++KJatmxpNDgAAAAAgcPnykVCQsJJX5ZXUFCg+Ph4I0EBAAAACDw+JxdPPvmk7r77bq1Zs8ZzbM2aNbr33nv11FNPGQ0OAAAAKGuWZdluC1SlaouqVKmS10NmZWWpRYsWKleu6OP5+fkqV66c/vGPf6h79+5lEigAAAAAeytVcvHcc8+VcRgAAAAAAl2pkouUlJSyjgMAAADwC7ut0GSnWHx1xi/Rk6Ts7Gzl5uZ6HXO5XH8pIAAAAACByecJ3VlZWRo8eLCqVaumiIgIVapUyWsDAAAAEJx8Ti4efPBBLVu2TJMnT5bT6dRrr72msWPHKj4+Xm+88UZZxAgAAACUGYdl2W4LVD63Rc2dO1dvvPGG2rRpo759+6ply5aqV6+eEhMTNXPmTPXp06cs4gQAAABgcz5XLg4dOqQ6depIKppfcejQIUnSVVddpZUrV5qNDgAAAEDA8Dm5qFOnjnbu3ClJatCggd555x1JRRWN6Ohoo8EBAAAAZa14tSg7bYHK5+Sib9+++uabbyRJDz30kCZNmqTw8HANGTJEw4YNMx4gAAAAgMDg85yLIUOGeP7cvn17fffdd1q7dq3q1aunJk2aGA0OAAAAQOD4S++5kKTExEQlJiaaiAUAAAA46yzLkmWjXiQ7xeKrUiUXL7zwQqkveM8995xxMAAAAAACV6mSi2effbZUF7Msyy/Jxdf/e0yRvBkcwDmiy6RV/g4BAIzJz87ydwg4i0qVXBSvDgUAAACcaxw6g1WOypCdYvFVIMcOAAAAwEZILgAAAAAY8ZdXiwIAAAACGatFmUPlAgAAAIARJBcAAAAAjDij5OLTTz/VLbfcouTkZP3888+SpDfffFOfffaZ0eAAAACAsmZZksNGWwB3RfmeXLz//vvq1KmTypcvr3Xr1iknJ0eSlJGRoccff9x4gAAAAAACg8/Jxb/+9S9NmTJFr776qkJDQz3Hr7zySn399ddGgwMAAAAQOHxeLWrr1q1q1apVieNRUVE6fPiwiZgAAACAs6a4Hcku7BSLr3yuXMTFxWn79u0ljn/22WeqU6eOkaAAAAAABB6fk4v+/fvr3nvv1ZdffinLsrRnzx7NnDlTDzzwgAYOHFgWMQIAAAAIAD63RT300EMqLCxUu3btdOzYMbVq1UpOp1MPPPCA7r777rKIEQAAACgzvETPHJ+TC8uy9Mgjj2jYsGHavn27jh49qqSkJFWsWLEs4gMAAAAQIHxOLoqFhYUpKSnJZCwAAAAAApjPyUXbtm3/tFSzbNmyvxQQAAAAcDaxWpQ5PicXzZo189rPy8vT+vXr9e233yolJcVUXAAAAAACjM/JxbPPPnvS42PGjNHRo0f/ckAAAAAAApPPS9Geyi233KLXX3/d1OUAAACAs8Ky7LcFKmPJRWpqqsLDw01dDgAAAECA8bktqkePHl77brdbe/fu1Zo1azRy5EhjgQEAAAAILD4nF1FRUV77DodD9evX17hx49SxY0djgQEAAABng8Oy5LBRL5KdYvGVT8lFQUGB+vbtq8aNG6tSpUplFRMAAACAAOTTnIuQkBB17NhRhw8fLqNwAAAAAAQqnyd0N2rUSD/88ENZxAIAAACcdQ4bboHK59j/9a9/6YEHHtC8efO0d+9eZWZmem0AAAAAglOpk4tx48YpKytL11xzjb755htdf/31qlGjhipVqqRKlSopOjqaeRgAAADAWTZhwgRdeumlioyMVLVq1dS9e3dt3brVa0x2drYGDRqkypUrq2LFiurZs6f27dvnNWb37t3q2rWrKlSooGrVqmnYsGHKz8/3KZZST+geO3as7rzzTn3yySc+3QAAAACwM7u9uM7XWFasWKFBgwbp0ksvVX5+vh5++GF17NhRmzdvVkREhCRpyJAhmj9/vt59911FRUVp8ODB6tGjhz7//HNJRQs3de3aVXFxcVq1apX27t2r2267TaGhoXr88cdLHUupkwu32y1Jat26tS/PCgAAAKAMLVy40Gt/+vTpqlatmtauXatWrVopIyNDU6dO1axZs3T11VdLkqZNm6aGDRvqiy++0OWXX66PP/5Ymzdv1pIlSxQbG6tmzZpp/PjxGj58uMaMGaOwsLBSxeLTnAvLTikdAAAAgBIyMjIkSTExMZKktWvXKi8vT+3bt/eMadCggWrWrKnU1FRJUmpqqho3bqzY2FjPmE6dOikzM1ObNm0q9b19es/FBRdccNoE49ChQ75cEgAAAPArh2z2Ej0VxfLHxZKcTqecTueffrawsFD33XefrrzySjVq1EiSlJ6errCwMEVHR3uNjY2NVXp6umfM7xOL4vPF50rLp+Ri7NixJd7QDQAAAMC8hIQEr/3Ro0drzJgxf/qZQYMG6dtvv9Vnn31WhpGdmk/JRa9evVStWrWyigUAAADACWlpaXK5XJ7901UtBg8erHnz5mnlypWqUaOG53hcXJxyc3N1+PBhr+rFvn37FBcX5xmzevVqr+sVryZVPKY0Sj3ngvkWAAAAOBcVrxZlp02SXC6X13aq5MLtdmvw4MGaPXu2li1bptq1a3udb968uUJDQ7V06VLPsa1bt2r37t1KTk6WJCUnJ2vjxo3av3+/Z8zixYvlcrmUlJRU6u+lz6tFAQAAALCPQYMGadasWfrwww8VGRnpmSMRFRWl8uXLKyoqSv369dPQoUMVExMjl8ulu+++W8nJybr88sslSR07dlRSUpJuvfVWTZw4Uenp6Xr00Uc1aNCg01ZMfq/UyUVhYaGPjwkAAACgrE2ePFmS1KZNG6/j06ZN0+233y5JevbZZ+VwONSzZ0/l5OSoU6dOevnllz1jQ0JCNG/ePA0cOFDJycmKiIhQSkqKxo0b51MsPs25AAAAAM41DqtoswtfYylNh1F4eLgmTZqkSZMmnXJMYmKiFixY4NvN/8Cn91wAAAAAwKmQXAAAAAAwgrYoAAAABDXLkq1eomejUHxG5QIAAACAESQXAAAAAIygLQoAAABB7fcvrrMDO8XiKyoXAAAAAIwguQAAAABgBG1RAAAACGqB/hI9O6FyAQAAAMAIkgsAAAAARtAWBQAAgKBmnfiyCzvF4isqFwAAAACMILkAAAAAYARtUQAAAAhqrBZlDpULAAAAAEaQXAAAAAAwgrYoAAAABDXaosyhcgEAAADACJILAAAAAEbQFgUAAICgZlmWLMs+vUh2isVXVC4AAAAAGEFyAQAAAMAI2qIAAAAQ1FgtyhwqFwAAAACMILkAAAAAYARtUQAAAAhqllW02YWdYvEVlQsAAAAARpBcAAAAADCCtigAAAAENYdlyWGjXiQ7xeIrKhcAAAAAjCC5AAAAAGAEbVEAAAAIarxEzxwqFwAAAACMILkAAAAAYARtUQAAAAhuNnuJnuwUi4+oXAAAAAAwguQCAAAAgBG0RQEAACCoOWTJYaNeJDvF4isqFwAAAACMILkAAAAAYARtUQAAAAhqls1Wi7JTLL6icgEAAADACJILAAAAAEbQFgUAAICg5rCKNruwUyy+onIBAAAAwAiSCwAAAABG0BYFAACAoOawLDlstESTnWLxFZULAAAAAEaQXAAAAAAwgrYoAAAABDVeomcOlQsAAAAARpBcAAAAADCCtigAAAAENYdstlqU7BOLr6hcAAAAADCC5AIAAACAEbRFAQAAIKixWpQ5VC4AAAAAGEFyAQAAAMAI2qIAAAAQ1Byy12/c7RSLrwI5dgAAAAA2QnIBAAAAwAjaogAAABDULMuSZaMlmuwUi6+oXAAAAAAwguQCAAAAgBG0RQEAACCoWSc2u7BTLL6icgEAAADACJILAAAAAEbQFgUAAICg5rAsOWy0QpOdYvEVlQsAAAAARpBcAAAAADCCtigAAAAEvcBtRLIXKhcAAAAAjCC5AAAAAGAEbVEAAAAIapZVtNmFnWLxFZULAAAAAEaQXAAAAAAwgrYoAAAABDXLsmTZqBfJTrH4isoFAAAAACNILgAAAAAYQVsUAAAAgppD9vqNu51i8VUgxw4AAADARkguAAAAABhBWxQAAACCGqtFmUPlAgAAAIARJBcAAAAAjKAtCgAAAEHNOrHZhZ1i8RWVCwAAAABGkFwAAAAAMIK2KAAAAAQ1Vosyh8oFAAAAACNILgAAAAAYQVsUAAAAgppD9vqNu51i8VUgxw4AAADARkguAAAAABhBcgEAAICgVrxalJ02X61cuVLXXXed4uPjZVmW5syZ43X+9ttvL3GPzp07e405dOiQ+vTpI5fLpejoaPXr109Hjx71KQ6SCwAAACDAZWVlqWnTppo0adIpx3Tu3Fl79+71bP/973+9zvfp00ebNm3S4sWLNW/ePK1cuVIDBgzwKQ4mdAMAAAABrkuXLurSpcufjnE6nYqLizvpuS1btmjhwoX66quvdMkll0iSXnzxRV1zzTV66qmnFB8fX6o4qFwAAAAgqFk23MrC8uXLVa1aNdWvX18DBw7UwYMHPedSU1MVHR3tSSwkqX379nI4HPryyy9LfQ8qFwAAAIANZWZmeu07nU45nc4zulbnzp3Vo0cP1a5dWzt27NDDDz+sLl26KDU1VSEhIUpPT1e1atW8PlOuXDnFxMQoPT291PchuQAAAABsKCEhwWt/9OjRGjNmzBldq1evXp4/N27cWE2aNFHdunW1fPlytWvX7q+E6YXkAgAAAEHNsoo2uyiOJS0tTS6Xy3P8TKsWJ1OnTh1VqVJF27dvV7t27RQXF6f9+/d7jcnPz9ehQ4dOOU/jZEguENRSP1uhm7p1Ou24oQ+N1H3DHpEk7fk5TUsWLtD6r7/S+rVfacf27+V2u/X2h4uUfFXrsg4ZAOQs59AlNaN1RZ1KahTvUlykUwVut/YcztbK7Qf1zro9ys4rPOlnQxyWujeJU7v6VVSzUnk5LEu/ZOXq2z2Zej01Tb9k5XrGxkY6dUWdSrq8ViXVqxohV3g5ZeUWaOu+o/pwQ7pW7fz1bD0yEJRcLpdXcmHSTz/9pIMHD6p69eqSpOTkZB0+fFhr165V8+bNJUnLli1TYWGhWrRoUerrklwgqFWtFqcbet1y0nMFBQWa/W7REm2XXX6l5/iCuXM07pFhZyU+ADiZdvWraFj7epKkXQePadUPh1QhLEQXVo9U3+Saurp+Fd333iYdPp7n9blIZzk9+bck1Y+tqF+O5mptWoYk6bzocHW5MFYLNu/3Si4e6Xy+Gse7lJtfqM3pR3QoK0/Vo5y6rFYlXVarkt79eo9e/nTXWXtuAKd29OhRbd++3bO/c+dOrV+/XjExMYqJidHYsWPVs2dPxcXFaceOHXrwwQdVr149depU9EvWhg0bqnPnzurfv7+mTJmivLw8DR48WL169Sr1SlESyQWCXL0L6uuZSa+d9NwnSxZp9rv/Vfx5NbwqEjUTa6vfnXer6UXN1fSiSzRy+H1a+cmSsxUyAKig0K25G9P13rq92v3rcc/xmAqhmtCtoS6oVlGDW9fSvxZu8/rcmK71VT+2oqZ/kaY3V6ep0P3bueoup7JyC7zGHziSq+c/+UGLtuzX8d9VQi6vVUnjr62vv18cr9U//qo1uzPK5kGBs8QhS44yW6PJd2cSy5o1a9S2bVvP/tChQyVJKSkpmjx5sjZs2KAZM2bo8OHDio+PV8eOHTV+/HivVquZM2dq8ODBateunRwOh3r27KkXXnjBpzhILoBTmP3uLElS9xt6eb0ps2OXa9Wxy7We/TN5iyYA/BWLthzQoi0HShw/dCxPz3/ygybd1EQt61ZWOcd25Z/IINqcX1kXJ0Rp+fe/aMaXaSU+uzczp8Sx8Qu/P+n9v9j1qz7avF/XNY7T1fWrklwANtCmTRu53e5Tnl+0aNFprxETE6NZs2b9pTh4zwVwEseysvTxR/MkST1u7OPnaACg9Hb8ckySFFbOIVf4b79D7NooVpL0wTd7jd6nSkSokesBODdQuQBO4qN5c3QsK0uNmjTTBQ0a+jscACi16lHhkqS8gkIdycmXVDSJu3F8pPILCvXdvqOqU6WCWterrEoVQvXL0Vx9/sMhT7JQ6vu4ilopDmXlnWYkYH92XS0qEJFcACdRPJG7x429/RwJAPimZ7OilV9W/3hYeQVFLRLxLqec5UJ0KCtXN1wUr37JNRXi+O2nl5TLE/T++r16eeWuUt0jIixEHRsWvWzr8x8OmX0AAAGN5AL4g33pe/X5yk8UEhKi63vc6O9wAKDUWtSK1jUXVlNeQaGmpe72HK94oj3KFV5OA65M1Jxv9uqdr/coK7dAV9aJ0d1tauvvF8Vrz+Fszdlw+jfxDr26ripVCNWmvUf06Q6SCwC/Yc4F8Af/++AdFRQUqGWbdqoWW/qXxgCAPyVUKq+HO50vh2Xplc9+9GpzcpzosSgX4tAXu37V88t3am9mjjKz8/XR5v165bMfJUm9LznvtPfp1fw8XV2/ijKO5+mxU0z4BgKNZcOvQEVyAfwBLVEAAk2ViDBN7N5QrvBQvfP1Hr2/3nvS9vG835aYXbhp/x8/roWbi45VjXQq/sScjZNpX7+K+l9ZU8dzCzTif1tOusIUgOBGcgH8zrat3+nbDesVEVFRna653t/hAMBpFb8YL84Vro827dPkk7zUbt/vkoD0IyUTgpz8Qh06VvTyvEoVTr760+W1K2l4h3rKL3Br1PzvtCX9qJkHAHBOYc4F8DsfvDNTktT52m4qX6GCn6MBgD8XHurQv7s3VK3KFbRy+0E9tXTHScdl5RZoT0a24qPCFekMKXHeklQxrOhHgt9XOYo1Pc+lMddcIMuy9NjCrbzXAuccVosyx6+Vi5UrV+q6665TfHy8LMvSnDlz/BkOgpzb7daH778tiZYoAPYXGmLpsesaqGFcpFbv+lXjP/re643bf7TqxKpOzWpElTiXVD1SYeUcys4rUNrv3vgtSedXjdC/rmug0BCHnlyyXSu3M4EbwKn5NbnIyspS06ZNNWnSJH+GAUiSVqd+pp/Sdiuu+nm6slVbf4cDAKfksKRHO1+gixOi9c3PGRo1f6vnTdyn8t66vcrNL1T3JnFqGFfRc9wVXk6DWtWSVDT3onj5WklKiA7Xv7snqaKznF5asfOkbwUHgN/za1tUly5d1KVLF3+GAHh8cGIid7cbbpLDceq8e1/6Xg247SbP/o5tWyVJjwy7V5GRLknS1R07694HHi7DaAEEs781ra5W9SpLkjKO5+u+tnVOOm7yp7uUmV30Ir19R3L03Cc/6IH2dfX8DY20ee8RHc0tUKPqkYoqH6rv9x3VK5//6PX5kV0uUKUKofr1WJ4uqFZRwzvUK3GP3b8e13/X/Gz4CYGzy5Ilh41WaArk1aICas5FTk6OcnJ+m4iWmZnpx2hwLsnJydGC/82WJPX4+81/OjY3N0fr1q4ucXz79995/lz3/AvMBggAv1PR+dv/vouTjJOZ/kWaJ7mQpI8279eejGz1vuQ8NYyLlLOcQ3sysvXBN3v1zto9ys4v9Pp85In3Y1SqEKrOSdVOeo/1P2WQXADwCKjkYsKECRo7dqy/w8A5yOl0auOOvacfKCmhZi3tPphdxhEBwKnN+DJNM75MO6PPfvNzpr75uXS/nLt52tdndA8AwSuglqIdMWKEMjIyPFta2pn9xQoAAAAUK14tyk5boAqoyoXT6ZTT6fR3GAAAAABOIqAqFwAAAADsy6+Vi6NHj2r79u2e/Z07d2r9+vWKiYlRzZo1/RgZAAAAgoXdWpHsFIuv/JpcrFmzRm3b/vY+gaFDh0qSUlJSNH36dD9FBQAAAOBM+DW5aNOmjdzuP3/pDwAAAIDAEFATugEAAADTrBNfdmGnWHzFhG4AAAAARpBcAAAAADCCtigAAAAENYdVtNmFnWLxFZULAAAAAEaQXAAAAAAwgrYoAAAABDVWizKHygUAAAAAI0guAAAAABhBWxQAAACCmmUVbXZhp1h8ReUCAAAAgBEkFwAAAACMoC0KAAAAQc2SvVZosk8kvqNyAQAAAMAIkgsAAAAARtAWBQAAgKDmsIo2u7BTLL6icgEAAADACJILAAAAAEbQFgUAAICgZp34sgs7xeIrKhcAAAAAjCC5AAAAAGAEbVEAAAAIapZVtNmFnWLxFZULAAAAAEaQXAAAAAAwgrYoAAAABDXrxGYXdorFV1QuAAAAABhBcgEAAADACNqiAAAAENQcsuSw0RJNjgBujKJyAQAAAMAIkgsAAAAARpBcAAAAADCCORcAAAAIaixFaw6VCwAAAABGkFwAAAAAMIK2KAAAAAQ3+qKMoXIBAAAAwAiSCwAAAABG0BYFAACAoGad+LILO8XiKyoXAAAAAIwguQAAAABgBG1RAAAACG6WZNmpE8lOsfiIygUAAAAAI0guAAAAABhBWxQAAACCGu/QM4fKBQAAAAAjSC4AAAAAGEFbFAAAAIIbfVHGULkAAAAAYATJBQAAAAAjaIsCAABAULNOfNmFnWLxFZULAAAAAEaQXAAAAAAwgrYoAAAABDXLKtrswk6x+IrKBQAAAAAjSC4AAAAAGEFbFAAAAIIa79Azh8oFAAAAACNILgAAAAAYQVsUAAAAght9UcZQuQAAAABgBMkFAAAAACNoiwIAAEBQs0582YWdYvEVlQsAAAAARpBcAAAAADCCtigAAAAENcsq2uzCTrH4isoFAAAAACNILgAAAAAYQVsUAAAAghrv0DOHygUAAAAAI0guAAAAABhBWxQAAACCG31RxlC5AAAAAGAEyQUAAAAAI2iLAgAAQFCzTnzZhZ1i8RWVCwAAAABGkFwAAAAAMILkAgAAAEHNsuy3+WrlypW67rrrFB8fL8uyNGfOHK/zbrdbo0aNUvXq1VW+fHm1b99e27Zt8xpz6NAh9enTRy6XS9HR0erXr5+OHj3qUxwkFwAAAECAy8rKUtOmTTVp0qSTnp84caJeeOEFTZkyRV9++aUiIiLUqVMnZWdne8b06dNHmzZt0uLFizVv3jytXLlSAwYM8CkOJnQDAAAAAa5Lly7q0qXLSc+53W4999xzevTRR9WtWzdJ0htvvKHY2FjNmTNHvXr10pYtW7Rw4UJ99dVXuuSSSyRJL774oq655ho99dRTio+PL1UcVC4AAAAQ1Cwbbibt3LlT6enpat++vedYVFSUWrRoodTUVElSamqqoqOjPYmFJLVv314Oh0Nffvllqe9F5QIAAACwoczMTK99p9Mpp9Pp83XS09MlSbGxsV7HY2NjPefS09NVrVo1r/PlypVTTEyMZ0xpULkAAAAAbCghIUFRUVGebcKECf4O6bSoXAAAACC4lUUv0l9xIpa0tDS5XC7P4TOpWkhSXFycJGnfvn2qXr265/i+ffvUrFkzz5j9+/d7fS4/P1+HDh3yfL40qFwAAAAANuRyuby2M00uateurbi4OC1dutRzLDMzU19++aWSk5MlScnJyTp8+LDWrl3rGbNs2TIVFhaqRYsWpb4XlQsAAAAgwB09elTbt2/37O/cuVPr169XTEyMatasqfvuu0//+te/dP7556t27doaOXKk4uPj1b17d0lSw4YN1blzZ/Xv319TpkxRXl6eBg8erF69epV6pSiJ5AIAAABBzjrxZRdnEsuaNWvUtm1bz/7QoUMlSSkpKZo+fboefPBBZWVlacCAATp8+LCuuuoqLVy4UOHh4Z7PzJw5U4MHD1a7du3kcDjUs2dPvfDCCz7FQXIBAAAABLg2bdrI7Xaf8rxlWRo3bpzGjRt3yjExMTGaNWvWX4qDORcAAAAAjKByAQAAgKBmWUWbXdgpFl9RuQAAAABgBMkFAAAAACNoiwIAAEBQs+k79AISlQsAAAAARpBcAAAAADCCtigAAAAEN/qijKFyAQAAAMAIkgsAAAAARtAWBQAAgKBmnfiyCzvF4isqFwAAAACMILkAAAAAYARtUQAAAAhqllW02YWdYvEVlQsAAAAARpBcAAAAADCCtigAAAAENd6hZw6VCwAAAABGkFwAAAAAMIK2KAAAAAQ3+qKMoXIBAAAAwAiSCwAAAABG0BYFAACAoGad+LILO8XiKyoXAAAAAIwguQAAAABgBG1RAAAACG6WZNmpE8lOsfiIygUAAAAAI0guAAAAABhBWxQAAACCGu/QM4fKBQAAAAAjSC4AAAAAGEFbFAAAAIIbfVHGULkAAAAAYATJBQAAAAAjaIsCAABAULNOfNmFnWLxFZULAAAAAEaQXAAAAAAwgrYoAAAABDXLKtrswk6x+IrKBQAAAAAjSC4AAAAAGEFbFAAAAIIa79Azh8oFAAAAACNILgAAAAAYQVsUAAAAght9UcZQuQAAAABgBMkFAAAAACNoiwIAAEBQs0582YWdYvEVlQsAAAAARpBcAAAAADCCtigAAAAENUuSZaNOJBuF4jMqFwAAAACMILkAAAAAYARtUQAAAAhqvEPPHCoXAAAAAIwguQAAAABgBG1RAAAACGqWZbPVomwUi6+oXAAAAAAwguQCAAAAgBG0RQEAACDIsV6UKVQuAAAAABgR0JULt9stSTp65IifIwEAc/Kzs/wdAgAYU/x3WvHPbTi3BXRyceREUtGiSV0/RwIAAIA/c+TIEUVFRfk7jJNitShzAjq5iI+PV1pamiIjI2UF8r8F2F5mZqYSEhKUlpYml8vl73AA4C/j7zWcLW63W0eOHFF8fLy/Q8FZENDJhcPhUI0aNfwdBoKIy+Xif8IAzin8vYazwa4VC5gX0MkFAAAA8FexVpQ5rBYFAAAAwAiSC6AUnE6nRo8eLafT6e9QAMAI/l4DUBYsN+uCAQAAIAhlZmYqKipKW3cfUKSN5h4dycxU/ZpVlZGREXBzoqhcAAAAADCC5AIAAACAEawWBQAAgKBmnfiyCzvF4isqFwAAAACMILkASmHSpEmqVauWwsPD1aJFC61evdrfIQHAGVm5cqWuu+46xcfHy7IszZkzx98hATiHkFwAp/H2229r6NChGj16tL7++ms1bdpUnTp10v79+/0dGgD4LCsrS02bNtWkSZP8HQpgH5YNtwDFUrTAabRo0UKXXnqpXnrpJUlSYWGhEhISdPfdd+uhhx7yc3QAcOYsy9Ls2bPVvXt3f4cC+EXxUrTfp/1iu6VoL0iowlK0wLkmNzdXa9euVfv27T3HHA6H2rdvr9TUVD9GBgAAYD8kF8Cf+OWXX1RQUKDY2Fiv47GxsUpPT/dTVAAAwCR/d0CdQ11RJBcAAAAAzCC5AP5ElSpVFBISon379nkd37dvn+Li4vwUFQAAgD2RXAB/IiwsTM2bN9fSpUs9xwoLC7V06VIlJyf7MTIAAGCKZdlvC1S8oRs4jaFDhyolJUWXXHKJLrvsMj333HPKyspS3759/R0aAPjs6NGj2r59u2d/586dWr9+vWJiYlSzZk0/RgbgXEByAZzGTTfdpAMHDmjUqFFKT09Xs2bNtHDhwhKTvAEgEKxZs0Zt27b17A8dOlSSlJKSounTp/spKgDnCt5zAQAAgKBU/J6LHT8dtN17LurWqMx7LgAAAAAEL5ILAAAAAEYw5wIAAADBzW5vrrNTLD6icgEAAADACJILAAAAAEbQFgUAAICgRleUOVQuAAAAABhBcgEAPrr99tvVvXt3z36bNm103333nfU4li9fLsuydPjw4VOOsSxLc+bMKfU1x4wZo2bNmv2luHbt2iXLsrR+/fq/dB0AQOAhuQBwTrj99ttlWZYsy1JYWJjq1auncePGKT8/v8zv/cEHH2j8+PGlGluahAAAcHZZlv22QMWcCwDnjM6dO2vatGnKycnRggULNGjQIIWGhmrEiBElxubm5iosLMzIfWNiYoxcBwCAQEflAsA5w+l0Ki4uTomJiRo4cKDat2+v//3vf5J+a2V67LHHFB8fr/r160uS0tLSdOONNyo6OloxMTHq1q2bdu3a5blmQUGBhg4dqujoaFWuXFkPPvig3G63133/2BaVk5Oj4cOHKyEhQU6nU/Xq1dPUqVO1a9cutW3bVpJUqVIlWZal22+/XZJUWFioCRMmqHbt2ipfvryaNm2q9957z+s+CxYs0AUXXKDy5curbdu2XnGW1vDhw3XBBReoQoUKqlOnjkaOHKm8vLwS41555RUlJCSoQoUKuvHGG5WRkeF1/rXXXlPDhg0VHh6uBg0a6OWXX/Y5FgDAuYfKBYBzVvny5XXw4EHP/tKlS+VyubR48WJJUl5enjp16qTk5GR9+umnKleunP71r3+pc+fO2rBhg8LCwvT0009r+vTpev3119WwYUM9/fTTmj17tq6++upT3ve2225TamqqXnjhBTVt2lQ7d+7UL7/8ooSEBL3//vvq2bOntm7dKpfLpfLly0uSJkyYoP/7v//TlClTdP7552vlypW65ZZbVLVqVbVu3VppaWnq0aOHBg0apAEDBmjNmjW6//77ff6eREZGavr06YqPj9fGjRvVv39/RUZG6sEHH/SM2b59u9555x3NnTtXmZmZ6tevn+666y7NnDlTkjRz5kyNGjVKL730ki666CKtW7dO/fv3V0REhFJSUnyOCQD8z5JlqzWa7BSLj9wAcA5ISUlxd+vWze12u92FhYXuxYsXu51Op/uBBx7wnI+NjXXn5OR4PvPmm2+669ev7y4sLPQcy8nJcZcvX969aNEit9vtdlevXt09ceJEz/m8vDx3jRo1PPdyu93u1q1bu++991632+12b9261S3JvXjx4pPG+cknn7gluX/99VfPsezsbHeFChXcq1at8hrbr18/98033+x2u93uESNGuJOSkrzODx8+vMS1/kiSe/bs2ac8/+STT7qbN2/u2R89erQ7JCTE/dNPP3mOffTRR26Hw+Heu3ev2+12u+vWreueNWuW13XGjx/vTk5OdrvdbvfOnTvdktzr1q075X0BwA4yMjLcktw79xxyHzyab5tt555DbknujIyMUj3H6NGj3ZK8tvr163vOHz9+3H3XXXe5Y2Ji3BEREe4ePXq409PTy+R7SuUCwDlj3rx5qlixovLy8lRYWKjevXtrzJgxnvONGzf2mmfxzTffaPv27YqMjPS6TnZ2tnbs2KGMjAzt3btXLVq08JwrV66cLrnkkhKtUcXWr1+vkJAQtW7dutRxb9++XceOHVOHDh28jufm5uqiiy6SJG3ZssUrDklKTk4u9T2Kvf3223rhhRe0Y8cOHT16VPn5+XK5XF5jatasqfPOO8/rPoWFhdq6dasiIyO1Y8cO9evXT/379/eMyc/PV1RUlM/xAADMuPDCC7VkyRLPfrlyv/2YP2TIEM2fP1/vvvuuoqKiNHjwYPXo0UOff/658ThILgCcM9q2bavJkycrLCxM8fHxXn+xSlJERITX/tGjR9W8eXNPu8/vVa1a9YxiKG5z8sXRo0clSfPnz/f6oV4qmkdiSmpqqvr06aOxY8eqU6dOioqK0ltvvaWnn37a51hfffXVEslOSEiIsVgB4Gyy2wpNZxJLuXLlFBcXV+J4RkaGpk6dqlmzZnlaeqdNm6aGDRvqiy++0OWXX/5Xw/WOw+jVAMCPIiIiVK9evVKPv/jii/X222+rWrVqJX57X6x69er68ssv1apVK0lFv6Ffu3atLr744pOOb9y4sQoLC7VixQq1b9++xPniyklBQYHnWFJSkpxOp3bv3n3KikfDhg09k9OLffHFF6d/yN9ZtWqVEhMT9cgjj3iO/fjjjyXG7d69W3v27FF8fLznPg6HQ/Xr11dsbKzi4+P1ww8/qE+fPj7dHwBQdrZt26b4+HiFh4crOTlZEyZMUM2aNbV27Vrl5eV5/T+pQYMGqlmzplJTU40nF6wWBSBo9enTR1WqVFG3bt306aefaufOnVq+fLnuuece/fTTT5Kke++9V0888YTmzJmj7777TnfdddefvqOiVq1aSklJ0T/+8Q/NmTPHc8133nlHkpSYmCjLsjRv3jwdOHBAR48eVWRkpB544AENGTJEM2bM0I4dO/T111/rxRdf1IwZMyRJd955p7Zt26Zhw4Zp69atmjVrlqZPn+7T855//vnavXu33nrrLe3YsUMvvPCCZs+eXWJceHi4UlJS9M033+jTTz/VPffcoxtvvNHzG7GxY8dqwoQJeuGFF/T9999r48aNmjZtmp555hmf4gEA/LnMzEyvLScn56TjWrRooenTp2vhwoWaPHmydu7cqZYtW+rIkSNKT09XWFiYoqOjvT4TGxur9PR04zGTXAAIWhUqVNDKlStVs2ZN9ejRQw0bNlS/fv2UnZ3tqWTcf//9uvXWW5WSkqLk5GRFRkbqb3/7259ed/Lkybrhhht01113qUGDBurfv7+ysrIkSeedd57Gjh2rhx56SLGxsRo8eLAkafz48Ro5cqQmTJighg0bqnPnzpo/f75q164tqWgexPvvv685c+aoadOmmjJlih5//HGfnvf666/XkCFDNHjwYDVr1kyrVq3SyJEjS4yrV6+eevTooWuuuUYdO3ZUkyZNvJaaveOOO/Taa69p2rRpaty4sVq3bq3p06d7YgUAmJGQkKCoqCjPNmHChJOO69Kli/7+97+rSZMm6tSpkxYsWKDDhw97frF1NlnuU81KBAAAAM5hmZmZioqK0q69h07ZHusPmZmZqlU9RmlpaV5xOZ3OUs/Fu/TSS9W+fXt16NBB7dq106+//upVvUhMTNR9992nIUOGGI2dygUAAABgQy6Xy2srbWJx9OhR7dixQ9WrV1fz5s0VGhqqpUuXes5v3bpVu3fvPqNVB0+HCd0AAAAIaoG+WtQDDzyg6667TomJidqzZ49Gjx6tkJAQ3XzzzYqKilK/fv00dOhQxcTEyOVy6e6771ZycrLxydwSyQUAAAAQ0H766SfdfPPNOnjwoKpWraqrrrpKX3zxhWdZ9WeffVYOh0M9e/ZUTk6OOnXq5DWXziTmXAAAACAoFc+5+DHdfnMuEuNilJGRYau4SoPKBQAAAIKadeLLLuwUi6+Y0A0AAADACJILAAAAAEbQFgUAAICgFuirRdkJlQsAAAAARpBcAAAAADCCtigAAAAENevEZhd2isVXVC4AAAAAGEFyAQAAAMAI2qIAAAAQ3OiLMobKBQAAAAAjSC4AAAAAGEFbFAAAAIKadeLLLuwUi6+oXAAAAAAwguQCAAAAgBG0RQEAACCoWVbRZhd2isVXVC4AAAAAGEFyAQAAAMAI2qIAAAAQ1HiHnjlULgAAAAAYQXIBAAAAwAjaogAAABDc6IsyhsoFAAAAACNILgAAAAAYQVsUAAAAgpp14ssu7BSLr6hcAAAAADCC5AIAAACAEbRFAQAAIKhZVtFmF3aKxVdULgAAAAAYQeUCAAAAQS0zM9PfIXixWzy+ILkAAABAUAoLC1NcXJzOr53g71BKiIuLU1hYmL/D8Jnldrvd/g4CAAAA8Ifs7Gzl5ub6O4wSwsLCFB4e7u8wfEZyAQAAAMAIJnQDAAAAMILkAgAAAIARJBcAAAAAjCC5AAAAAGAEyQUAAAAAI0guAAAAABhBcgEAAADAiP8Hn+oecGL3JHQAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "syK86rQQk977"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}