kal2257_7B.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# GPT for Sequence Classification"
      ],
      "metadata": {
        "id": "j5MHM1LuYWGK"
      },
      "id": "j5MHM1LuYWGK"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "85c15f62",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "85c15f62",
        "outputId": "7e6de168-c412-4c51-ca63-f1b0181582b1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting torchmetrics\n",
            "  Downloading torchmetrics-1.2.1-py3-none-any.whl (806 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m806.1/806.1 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>1.20.0 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (1.23.5)\n",
            "Requirement already satisfied: packaging>17.1 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (23.2)\n",
            "Requirement already satisfied: torch>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (2.1.0+cu121)\n",
            "Collecting lightning-utilities>=0.8.0 (from torchmetrics)\n",
            "  Downloading lightning_utilities-0.10.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from lightning-utilities>=0.8.0->torchmetrics) (67.7.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from lightning-utilities>=0.8.0->torchmetrics) (4.5.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (3.13.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (2.1.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.8.1->torchmetrics) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8.1->torchmetrics) (1.3.0)\n",
            "Installing collected packages: lightning-utilities, torchmetrics\n",
            "Successfully installed lightning-utilities-0.10.0 torchmetrics-1.2.1\n"
          ]
        }
      ],
      "source": [
        "# Imports\n",
        "!pip install torchmetrics\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertTokenizer, BertForSequenceClassification\n",
        "from transformers import GPT2Tokenizer, GPT2ForSequenceClassification, GPT2Config\n",
        "from sklearn.model_selection import train_test_split\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from tabulate import tabulate\n",
        "from tqdm import trange\n",
        "import random\n",
        "from torchmetrics.classification import Recall, Accuracy, AUROC, Precision\n",
        "from torch.utils.data.dataset import random_split\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dd3d5e84",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dd3d5e84",
        "outputId": "1c1ad4d6-45b3-4fbc-e835-fcf650395203"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ham\tGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
            "ham\tOk lar... Joking wif u oni...\n",
            "spam\tFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's\n",
            "ham\tU dun say so early hor... U c already then say...\n",
            "ham\tNah I don't think he goes to usf, he lives around here though\n",
            "spam\tFreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some fun you up for it still? Tb ok! XxX std chgs to send, £1.50 to rcv\n",
            "ham\tEven my brother is not like to speak with me. They treat me like aids patent.\n",
            "ham\tAs per your request 'Melle Melle (Oru Minnaminunginte Nurungu Vettam)' has been set as your callertune for all Callers. Press *9 to copy your friends Callertune\n",
            "spam\tWINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.\n",
            "spam\tHad your mobile 11 months or more? U R entitled to Update to the latest colour mobiles with camera for Free! Call The Mobile Update Co FREE on 08002986030\n"
          ]
        }
      ],
      "source": [
        "# Pulling Data\n",
        "!wget 'https://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip'\n",
        "!unzip -o smsspamcollection.zip\n",
        "!head -10 SMSSpamCollection"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize some variables\n",
        "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
        "tokenizer.pad_token = '[PAD]'\n",
        "tokenizer.padding_side = \"left\" # Remember we want to use the last token's embedding to represent the entire sentence\n",
        "val_ratio = 0.2\n",
        "batch_size = 16 # Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\n",
        "EOS_token=1\n",
        "epochs = 2 # Recommended number of epochs: See: https://arxiv.org/pdf/1810.04805.pdf\n",
        "\n",
        "# Use torchmetrics to set up accuracy, recall, precision, and auroc\n",
        "accuracy = Accuracy(task = \"binary\")\n",
        "recall =  Recall(task = \"binary\")\n",
        "precision =  Precision(task = \"binary\")\n",
        "auroc =  AUROC(task = \"binary\")"
      ],
      "metadata": {
        "id": "UDAR_UswLVQR",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "bfca66f744e44ccb88ee35c6e6fd8216",
            "4d64f87edacc44c7909e7de414171d39",
            "2ca8104dc9604f028734432c9ddbdcb9",
            "4fbdaebd350e4479b71edcbb844ec6e3",
            "096e1599efd3476bb0d08705626b0db3",
            "3b050096800d4c09b4219bc29e3775f6",
            "4732039532964a9d8d68ae7a61a2ef9c",
            "5d963b1602a04587a863b0f9438cb624",
            "d13faef7a2884051ab05a2e4bd2e725b",
            "b2c88eec98954305bbf413f21c0a0d6b",
            "6c98b3cfe847490394bb714a96f85138",
            "8f6076d893b747d084e6b0e506fa9d87",
            "a262c1a55d664152b080c8d6c250f8d8",
            "5af870ba1e154dd88b90f36908e39496",
            "7dd4b33c4ee24c3b9341dfc94b528b62",
            "f2fb4959fadf4bb9aca566621315a1c8",
            "bc3bc5dbf5a14379a890559b03617b55",
            "e4a91a1d42f94956bb4e3a2121e747f0",
            "965ae73f65174df791660945c0dabc56",
            "0feb1f3d9fb04620818db04bafe17d91",
            "1b33577bba1a4d9db7a9e5aa39a191d6",
            "70daddf6b3dc4800832d48f453ba7035",
            "cbb286a2095d436ba4cfc457ab88783d",
            "40221a30e85e4d2f8249115b30895ff8",
            "12d53b9483094c9eb3e2beaef9714b01",
            "f977c8a9daac4711a20998b6c80fcec1",
            "2b5f9bb7cbb04b41b2389456954cc0dd",
            "98907b33ebf7422ba654f10b5a676944",
            "55e7f4b41e35442e9dc903f6ab0e1747",
            "5d79ed2d4dd940b981d530d3dba90670",
            "4ae1fdd1daf344709ef5a7f3a668e155",
            "3b52f849dde54f0599766a72eabef999",
            "de6713abc31943a0beb6f5faa90ee296",
            "c242923ec30549a2a3bad23fa4b687af",
            "6e61c4b3fa2b42aba968cd6abbbf7f37",
            "d24a45b4ca0247f89bb82b691bbd0446",
            "284561e25b6b4f1880aade7d3ccac975",
            "d2f3190a86344a1ebb75724bde870862",
            "c317e39add724518b4afbf958e6c674c",
            "d3210b7a98624df4af3b3da901dd18d6",
            "a1c8d07b1f1940c897c17ab9677f5e66",
            "270a758cc443479f8c53be764af9ac4f",
            "d95742023f1f4e75a2549ee38777097c",
            "e45f0c22c54a43358c461d09f8a051af"
          ]
        },
        "outputId": "46b0e556-902a-45c1-9977-4b02084d4141"
      },
      "id": "UDAR_UswLVQR",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bfca66f744e44ccb88ee35c6e6fd8216"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8f6076d893b747d084e6b0e506fa9d87"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cbb286a2095d436ba4cfc457ab88783d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c242923ec30549a2a3bad23fa4b687af"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Prepare the Vocabulary/Tokenize Text"
      ],
      "metadata": {
        "id": "JWJfMeKDYjmp"
      },
      "id": "JWJfMeKDYjmp"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "98a736a1",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "98a736a1",
        "outputId": "ce337f4e-1552-479b-aa60-465253545f6b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                               text\n",
              "0      0  Go until jurong point, crazy.. Available only ...\n",
              "1      0                    Ok lar... Joking wif u oni...\\n\n",
              "2      1  Free entry in 2 a wkly comp to win FA Cup fina...\n",
              "3      0  U dun say so early hor... U c already then say...\n",
              "4      0  Nah I don't think he goes to usf, he lives aro..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e2823dd0-9ef0-4b4d-83ac-ae620508b4d3\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>Ok lar... Joking wif u oni...\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e2823dd0-9ef0-4b4d-83ac-ae620508b4d3')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e2823dd0-9ef0-4b4d-83ac-ae620508b4d3 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e2823dd0-9ef0-4b4d-83ac-ae620508b4d3');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-a70b6b03-61e0-4c3a-9f49-647b2ef32e34\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a70b6b03-61e0-4c3a-9f49-647b2ef32e34')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-a70b6b03-61e0-4c3a-9f49-647b2ef32e34 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "file_path = 'SMSSpamCollection'\n",
        "df = pd.DataFrame({'label':int(), 'text':str()}, index = [])\n",
        "with open(file_path) as f:\n",
        "    for line in f.readlines():\n",
        "        split = line.split('\\t')\n",
        "        df = pd.concat([\n",
        "                df,\n",
        "                pd.DataFrame.from_dict({\n",
        "                    'label': [1 if split[0] == 'spam' else 0],\n",
        "                    'text': [split[1]]\n",
        "                })\n",
        "            ],\n",
        "            ignore_index=True\n",
        "        )\n",
        "df.head()\n",
        "\n",
        "text = df.text.values\n",
        "labels = df.label.values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c00a3e7e",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c00a3e7e",
        "outputId": "ac9ee8e5-ca90-48cc-b9fc-08fc3a4a75ca"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Awww dat is sweet! We can think of something to do he he! Have a nice time tonight ill probably txt u later cos im lonely :( xxx.\n",
            "\n",
            "╒════════════╤═════════════╕\n",
            "│ Tokens     │   Token IDs │\n",
            "╞════════════╪═════════════╡\n",
            "│ A          │          32 │\n",
            "├────────────┼─────────────┤\n",
            "│ www        │        2503 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġdat       │        4818 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġis        │         318 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġsweet     │        6029 │\n",
            "├────────────┼─────────────┤\n",
            "│ !          │           0 │\n",
            "├────────────┼─────────────┤\n",
            "│ ĠWe        │         775 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġcan       │         460 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġthink     │         892 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġof        │         286 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġsomething │        1223 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġto        │         284 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġdo        │         466 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġhe        │         339 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġhe        │         339 │\n",
            "├────────────┼─────────────┤\n",
            "│ !          │           0 │\n",
            "├────────────┼─────────────┤\n",
            "│ ĠHave      │        8192 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġa         │         257 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġnice      │        3621 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġtime      │         640 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġtonight   │        9975 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġill       │        2801 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġprobably  │        2192 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġt         │         256 │\n",
            "├────────────┼─────────────┤\n",
            "│ xt         │         742 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġu         │         334 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġlater     │        1568 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġcos       │        8615 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġim        │         545 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġlonely    │       21757 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġ:(        │       36147 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ġx         │        2124 │\n",
            "├────────────┼─────────────┤\n",
            "│ xx         │        5324 │\n",
            "├────────────┼─────────────┤\n",
            "│ .          │          13 │\n",
            "├────────────┼─────────────┤\n",
            "│ Ċ          │         198 │\n",
            "╘════════════╧═════════════╛\n"
          ]
        }
      ],
      "source": [
        "def print_rand_sentence():\n",
        "    '''Displays the tokens and respective IDs of a random text sample'''\n",
        "    index = random.randint(0, len(text)-1)\n",
        "    print(text[index])\n",
        "    table = np.array([tokenizer.tokenize(text[index]),\n",
        "                    tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text[index]))]).T\n",
        "    print(tabulate(table,\n",
        "                 headers = ['Tokens', 'Token IDs'],\n",
        "                 tablefmt = 'fancy_grid'))\n",
        "\n",
        "print_rand_sentence()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocessing(input_text, tokenizer):\n",
        "  '''\n",
        "  Returns <class transformers.tokenization_utils_base.BatchEncoding> with the following fields:\n",
        "    - input_ids: list of token ids\n",
        "    - token_type_ids: list of token type ids\n",
        "    - attention_mask: list of indices (0,1) specifying which tokens should considered by the model (return_attention_mask = True).\n",
        "  '''\n",
        "  # Use the tokenizer and the encode_plus methods to return the right data we'll need\n",
        "  # Set max_length = 32 and return_tokens = 'pt'\n",
        "  # Set other fields to the appropriate booleans needed\n",
        "  return tokenizer.encode_plus(input_text, max_length = 32 , return_tensors = 'pt')"
      ],
      "metadata": {
        "id": "BPdQ1QiXeALU"
      },
      "id": "BPdQ1QiXeALU",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e85d0895",
      "metadata": {
        "id": "e85d0895"
      },
      "outputs": [],
      "source": [
        "token_id = []\n",
        "attention_masks = []\n",
        "for sample in text:\n",
        "    encoding_dict = preprocessing(sample, tokenizer)\n",
        "    token_id.append(encoding_dict['input_ids'])\n",
        "    attention_masks.append(encoding_dict['attention_mask'])\n",
        "\n",
        "# # Gather all the torch_id, attention masks, and labels\n",
        "token_id = [token.squeeze() for token in token_id]\n",
        "attention_masks = [attention.squeeze() for attention in attention_masks]\n",
        "labels = labels"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def print_rand_sentence_encoding():\n",
        "    '''Displays tokens, token IDs and attention mask of a random text sample'''\n",
        "    index = random.randint(0, len(text) - 1)\n",
        "    tokens = tokenizer.tokenize(tokenizer.decode(token_id[index]))\n",
        "    print(tokens)\n",
        "    token_ids = [i.numpy() for i in token_id[index]]\n",
        "    attention = [i.numpy() for i in attention_masks[index]]\n",
        "    table = np.array([tokens, token_ids, attention]).T\n",
        "    print(\n",
        "        tabulate(\n",
        "            table,\n",
        "            headers = ['Tokens', 'Token IDs', 'Attention Mask'],\n",
        "            tablefmt = 'fancy_grid')\n",
        "    )\n",
        "\n",
        "print_rand_sentence_encoding()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SpYXYFEcQcvW",
        "outputId": "0ef7d955-ba24-4aed-dc38-2fbdf0ed73dd"
      },
      "id": "SpYXYFEcQcvW",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Hey', 'Ġleave', 'Ġit', '.', 'Ġnot', 'Ġa', 'Ġbig', 'Ġdeal', ':-', ')', 'Ġtake', 'Ġcare', '.', 'Ċ']\n",
            "╒══════════╤═════════════╤══════════════════╕\n",
            "│ Tokens   │   Token IDs │   Attention Mask │\n",
            "╞══════════╪═════════════╪══════════════════╡\n",
            "│ Hey      │       10814 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ Ġleave   │        2666 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ Ġit      │         340 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ .        │          13 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ Ġnot     │         407 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ Ġa       │         257 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ Ġbig     │        1263 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ Ġdeal    │        1730 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ :-       │       21912 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ )        │           8 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ Ġtake    │        1011 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ Ġcare    │        1337 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ .        │          13 │                1 │\n",
            "├──────────┼─────────────┼──────────────────┤\n",
            "│ Ċ        │         198 │                1 │\n",
            "╘══════════╧═════════════╧══════════════════╛\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e1c2c10b",
      "metadata": {
        "id": "e1c2c10b"
      },
      "outputs": [],
      "source": [
        "# Split the data into training and testing sets\n",
        "num_train = int(len(token_id) * 0.8)\n",
        "train, valid = random_split(token_id, [num_train, len(token_id) - num_train])\n",
        "train_dataloader = DataLoader(train, batch_size=batch_size, shuffle=True)\n",
        "valid_dataloader = DataLoader(valid, batch_size=batch_size, shuffle=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "40bcf8c4",
      "metadata": {
        "id": "40bcf8c4"
      },
      "source": [
        "## Define the model: GPT Transformer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f85fc88e",
      "metadata": {
        "id": "f85fc88e"
      },
      "outputs": [],
      "source": [
        "# Load the GPTForSequenceClassification model\n",
        "# Set to 'gpt2' (the smallest GPT2 which is 120 M parameters)\n",
        "# Do not ouput the attentions and all hidden states\n",
        "config =  GPT2Config()\n",
        "\n",
        "# Use the config above and set other labels as needed\n",
        "model = GPT2ForSequenceClassification(config)\n",
        "\n",
        "# Set the pad token id to the eos token id\n",
        "model.config.pad_token_id = EOS_token\n",
        "\n",
        "# Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5\n",
        "# See: https://arxiv.org/pdf/1810.04805.pdf\n",
        "optimizer = torch.optim.AdamW(\n",
        "    model.parameters(),\n",
        "    lr = 5e-5,\n",
        "    eps = 1e-08\n",
        ")\n",
        "# Set the model to the right device\n",
        "model = model.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Print all the layers of this GPT2 model and the number of parameters per layer\n",
        "fine_tune = True\n",
        "\n",
        "for name, m in model.named_modules():\n",
        "  for e in m.parameters():\n",
        "    if e.requires_grad==True:\n",
        "      print(name,e.numel())\n",
        "\n",
        "total_parameters = np.sum([param.numel() for param in model.parameters()] )\n",
        "assert(total_parameters == 124441344)"
      ],
      "metadata": {
        "id": "_Wlt5uot1wZo"
      },
      "id": "_Wlt5uot1wZo",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "id": "765fc3a1",
      "metadata": {
        "id": "765fc3a1"
      },
      "source": [
        "## Training the Model/ Model Execution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "084519f5",
      "metadata": {
        "id": "084519f5"
      },
      "outputs": [],
      "source": [
        "# Main training / validation loop\n",
        "for _ in trange(epochs, desc = 'Epoch'):\n",
        "\n",
        "    # ========== Training ==========\n",
        "\n",
        "    # Set model to training mode\n",
        "    model.train()\n",
        "\n",
        "    # Tracking variables\n",
        "    tr_loss = 0\n",
        "    nb_tr_examples, nb_tr_steps = 0, 0\n",
        "\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        # Put each element of batch onto the device\n",
        "        # Unpack the batch\n",
        "        batch = {k:v.type(torch.long).to(device_) for k,v in batch.items()}\n",
        "\n",
        "        # Set gradients to zero\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # Forward pass\n",
        "        train_output = model(batch**)\n",
        "\n",
        "        # Backward pass\n",
        "        loss =criterion(train_output,label)\n",
        "\n",
        "        # Do back propagation and get the gradients\n",
        "        loss.backward()\n",
        "\n",
        "\n",
        "        # Update tracking variables\n",
        "        tr_loss += loss\n",
        "        nb_tr_examples += len(labels)\n",
        "        nb_tr_steps += 1\n",
        "\n",
        "    # ========== Validation ==========\n",
        "\n",
        "    # Set model to evaluation mode\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    val_accuracy = []\n",
        "    val_precision = []\n",
        "    val_recall = []\n",
        "    val_auroc = []\n",
        "\n",
        "    for batch in validation_dataloader:\n",
        "        # Unpack the batch\n",
        "        batch = {k:v.type(torch.long).to(device_) for k,v in batch.items()}\n",
        "\n",
        "        with torch.no_grad():\n",
        "          # Forward pass\n",
        "          eval_output = model(batch**)\n",
        "\n",
        "        # Calculate validation metrics\n",
        "        loss, logits = outputs[:2]\n",
        "        labels = batch['labels'].numpy().flatten().tolist()\n",
        "        predicted_labels =logits.argmax(axis=-1).flatten().tolist()\n",
        "\n",
        "        val_accuracy.append(accuracy(predicted_labels,labels))\n",
        "        val_recall.append(recall(predicted_labels,labels))\n",
        "        val_precision.append(precision(predicted_labels,labels))\n",
        "        val_auroc.append(auroc(predicted_labels,labels))\n",
        "\n",
        "    print('\\n\\t - Train loss: {:.4f}'.format(tr_loss / nb_tr_steps))\n",
        "    print('\\t - Validation Accuracy: {:.4f}'.format(sum(val_accuracy)/len(val_accuracy)))\n",
        "    print('\\t - Validation Precision: {:.4f}'.format(sum(val_precision)/len(val_precision)))\n",
        "    print('\\t - Validation Recall: {:.4f}'.format(sum(val_recall)/len(val_recall)))\n",
        "    print('\\t - Validation AUROC: {:.4f}\\n'.format(sum(val_auroc)/len(val_auroc)))"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "13d64bc6",
      "metadata": {
        "id": "13d64bc6"
      },
      "source": [
        "## Model Validation:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6dfcd9e7",
      "metadata": {
        "id": "6dfcd9e7"
      },
      "outputs": [],
      "source": [
        "new_sentence = 'WINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.'\n",
        "\n",
        "# We need Token IDs and Attention Mask for inference on the new sentence\n",
        "test_ids = []\n",
        "test_attention_mask = []\n",
        "\n",
        "# Apply the tokenizer\n",
        "encoding = preprocessing(new_sentence, tokenizer)\n",
        "\n",
        "# Extract IDs and Attention Mask\n",
        "test_ids.append(encoding['input_ids'])\n",
        "test_attention_mask.append(encoding['attention_mask'])\n",
        "test_ids = torch.cat(test_ids, dim = 0)\n",
        "test_attention_mask = torch.cat(test_attention_mask, dim = 0)\n",
        "\n",
        "# Forward pass, calculate logit predictions\n",
        "with torch.no_grad():\n",
        "    output = model(test_ids.to(device), token_type_ids = None, attention_mask = test_attention_mask.to(device))\n",
        "\n",
        "prediction = 'Spam' if np.argmax(output.logits.cpu().numpy()).flatten().item() == 1 else 'Ham'\n",
        "\n",
        "print('Input Sentence: ', new_sentence)\n",
        "print('Predicted Class: ', prediction)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.15"
    },
    "colab": {
      "provenance": []
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bfca66f744e44ccb88ee35c6e6fd8216": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4d64f87edacc44c7909e7de414171d39",
              "IPY_MODEL_2ca8104dc9604f028734432c9ddbdcb9",
              "IPY_MODEL_4fbdaebd350e4479b71edcbb844ec6e3"
            ],
            "layout": "IPY_MODEL_096e1599efd3476bb0d08705626b0db3"
          }
        },
        "4d64f87edacc44c7909e7de414171d39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b050096800d4c09b4219bc29e3775f6",
            "placeholder": "​",
            "style": "IPY_MODEL_4732039532964a9d8d68ae7a61a2ef9c",
            "value": "vocab.json: 100%"
          }
        },
        "2ca8104dc9604f028734432c9ddbdcb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d963b1602a04587a863b0f9438cb624",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d13faef7a2884051ab05a2e4bd2e725b",
            "value": 1042301
          }
        },
        "4fbdaebd350e4479b71edcbb844ec6e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b2c88eec98954305bbf413f21c0a0d6b",
            "placeholder": "​",
            "style": "IPY_MODEL_6c98b3cfe847490394bb714a96f85138",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 11.7MB/s]"
          }
        },
        "096e1599efd3476bb0d08705626b0db3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3b050096800d4c09b4219bc29e3775f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4732039532964a9d8d68ae7a61a2ef9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5d963b1602a04587a863b0f9438cb624": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d13faef7a2884051ab05a2e4bd2e725b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b2c88eec98954305bbf413f21c0a0d6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c98b3cfe847490394bb714a96f85138": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8f6076d893b747d084e6b0e506fa9d87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a262c1a55d664152b080c8d6c250f8d8",
              "IPY_MODEL_5af870ba1e154dd88b90f36908e39496",
              "IPY_MODEL_7dd4b33c4ee24c3b9341dfc94b528b62"
            ],
            "layout": "IPY_MODEL_f2fb4959fadf4bb9aca566621315a1c8"
          }
        },
        "a262c1a55d664152b080c8d6c250f8d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc3bc5dbf5a14379a890559b03617b55",
            "placeholder": "​",
            "style": "IPY_MODEL_e4a91a1d42f94956bb4e3a2121e747f0",
            "value": "merges.txt: 100%"
          }
        },
        "5af870ba1e154dd88b90f36908e39496": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_965ae73f65174df791660945c0dabc56",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0feb1f3d9fb04620818db04bafe17d91",
            "value": 456318
          }
        },
        "7dd4b33c4ee24c3b9341dfc94b528b62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b33577bba1a4d9db7a9e5aa39a191d6",
            "placeholder": "​",
            "style": "IPY_MODEL_70daddf6b3dc4800832d48f453ba7035",
            "value": " 456k/456k [00:00&lt;00:00, 18.7MB/s]"
          }
        },
        "f2fb4959fadf4bb9aca566621315a1c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc3bc5dbf5a14379a890559b03617b55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4a91a1d42f94956bb4e3a2121e747f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "965ae73f65174df791660945c0dabc56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0feb1f3d9fb04620818db04bafe17d91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1b33577bba1a4d9db7a9e5aa39a191d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70daddf6b3dc4800832d48f453ba7035": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cbb286a2095d436ba4cfc457ab88783d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_40221a30e85e4d2f8249115b30895ff8",
              "IPY_MODEL_12d53b9483094c9eb3e2beaef9714b01",
              "IPY_MODEL_f977c8a9daac4711a20998b6c80fcec1"
            ],
            "layout": "IPY_MODEL_2b5f9bb7cbb04b41b2389456954cc0dd"
          }
        },
        "40221a30e85e4d2f8249115b30895ff8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_98907b33ebf7422ba654f10b5a676944",
            "placeholder": "​",
            "style": "IPY_MODEL_55e7f4b41e35442e9dc903f6ab0e1747",
            "value": "tokenizer.json: 100%"
          }
        },
        "12d53b9483094c9eb3e2beaef9714b01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d79ed2d4dd940b981d530d3dba90670",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4ae1fdd1daf344709ef5a7f3a668e155",
            "value": 1355256
          }
        },
        "f977c8a9daac4711a20998b6c80fcec1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b52f849dde54f0599766a72eabef999",
            "placeholder": "​",
            "style": "IPY_MODEL_de6713abc31943a0beb6f5faa90ee296",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 16.0MB/s]"
          }
        },
        "2b5f9bb7cbb04b41b2389456954cc0dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "98907b33ebf7422ba654f10b5a676944": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55e7f4b41e35442e9dc903f6ab0e1747": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5d79ed2d4dd940b981d530d3dba90670": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ae1fdd1daf344709ef5a7f3a668e155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3b52f849dde54f0599766a72eabef999": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de6713abc31943a0beb6f5faa90ee296": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c242923ec30549a2a3bad23fa4b687af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6e61c4b3fa2b42aba968cd6abbbf7f37",
              "IPY_MODEL_d24a45b4ca0247f89bb82b691bbd0446",
              "IPY_MODEL_284561e25b6b4f1880aade7d3ccac975"
            ],
            "layout": "IPY_MODEL_d2f3190a86344a1ebb75724bde870862"
          }
        },
        "6e61c4b3fa2b42aba968cd6abbbf7f37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c317e39add724518b4afbf958e6c674c",
            "placeholder": "​",
            "style": "IPY_MODEL_d3210b7a98624df4af3b3da901dd18d6",
            "value": "config.json: 100%"
          }
        },
        "d24a45b4ca0247f89bb82b691bbd0446": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a1c8d07b1f1940c897c17ab9677f5e66",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_270a758cc443479f8c53be764af9ac4f",
            "value": 665
          }
        },
        "284561e25b6b4f1880aade7d3ccac975": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d95742023f1f4e75a2549ee38777097c",
            "placeholder": "​",
            "style": "IPY_MODEL_e45f0c22c54a43358c461d09f8a051af",
            "value": " 665/665 [00:00&lt;00:00, 25.0kB/s]"
          }
        },
        "d2f3190a86344a1ebb75724bde870862": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c317e39add724518b4afbf958e6c674c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3210b7a98624df4af3b3da901dd18d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a1c8d07b1f1940c897c17ab9677f5e66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "270a758cc443479f8c53be764af9ac4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d95742023f1f4e75a2549ee38777097c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e45f0c22c54a43358c461d09f8a051af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}