Transformers - HKEX Annt Classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Transformers - HKEx Announcement Classifier.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X2rvzkc0Wjzu"
      },
      "source": [
        "## Multiclass Classifier for Announcements issued by Listed Companies on the Hong Kong Stock Exchange"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pbT9HLCf_SRj"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.models import Model, Sequential\n",
        "from tensorflow.keras.layers import Bidirectional, LSTM, Activation, Dense, Dropout, Input, Embedding, BatchNormalization, MultiHeadAttention, LayerNormalization, GlobalAveragePooling1D\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "import tensorflow as tf\n",
        "\n",
        "from plotly.subplots import make_subplots\n",
        "import plotly.graph_objects as go\n",
        "import plotly"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mFpvnZtN_vaX"
      },
      "source": [
        "# Get data from scraped CSVs\n",
        "CCT = '/content/drive/My Drive/Data/HKEX/ch14A_cct.csv'\n",
        "NT = '/content/drive/My Drive/Data/HKEX/notifiable_transactions.csv'\n",
        "TK = '/content/drive/My Drive/Data/HKEX/takeovers_code_3_7.csv'\n",
        "AGM = '/content/drive/My Drive/Data/HKEX/notice_of_agm.csv'\n",
        "TH = r'/content/drive/My Drive/Data/HKEX/trading halt.csv'\n",
        "AR = '/content/drive/My Drive/Data/HKEX/annual_results_annt.csv'\n",
        "\n",
        "ct = pd.read_csv(CCT)\n",
        "nt = pd.read_csv(NT)\n",
        "tk = pd.read_csv(TK)\n",
        "agm = pd.read_csv(AGM)\n",
        "th = pd.read_csv(TH)\n",
        "ar = pd.read_csv(AR)"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lcfr_xzp3gQf"
      },
      "source": [
        "# Trim data sources such that the amount of training examples from different categories are not vastly different\n",
        "agm = agm.sample(frac=1)\n",
        "agm = agm[:3000]\n",
        "ar = ar.sample(frac=1)\n",
        "ar = ar[:3000]"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 553
        },
        "id": "drXWpmTysMpI",
        "outputId": "eb667f53-0f3f-42fd-9ee9-2aaeaf73e7b0"
      },
      "source": [
        "# Concat all data sources\n",
        "df = ct.append(nt).append(tk).append(agm).append(th).append(ar)\n",
        "df"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>URL</th>\n",
              "      <th>Text</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Chapter 14A Connected Transaction Announcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Chapter 14A Connected Transaction Announcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Chapter 14A Connected Transaction Announcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong  Kong  Exchanges  and  Clearing  Limited ...</td>\n",
              "      <td>Chapter 14A Connected Transaction Announcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Chapter 14A Connected Transaction Announcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>814</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Annual Results Announcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2774</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Annual Results Announcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1921</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Annual Results Announcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>81</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/g...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Annual Results Announcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2153</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Annual Results Announcement</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>11933 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    URL  ...                                           Label\n",
              "0     https://www1.hkexnews.hk/listedco/listconews/s...  ...  Chapter 14A Connected Transaction Announcement\n",
              "1     https://www1.hkexnews.hk/listedco/listconews/s...  ...  Chapter 14A Connected Transaction Announcement\n",
              "2     https://www1.hkexnews.hk/listedco/listconews/s...  ...  Chapter 14A Connected Transaction Announcement\n",
              "3     https://www1.hkexnews.hk/listedco/listconews/s...  ...  Chapter 14A Connected Transaction Announcement\n",
              "4     https://www1.hkexnews.hk/listedco/listconews/s...  ...  Chapter 14A Connected Transaction Announcement\n",
              "...                                                 ...  ...                                             ...\n",
              "814   https://www1.hkexnews.hk/listedco/listconews/s...  ...                     Annual Results Announcement\n",
              "2774  https://www1.hkexnews.hk/listedco/listconews/s...  ...                     Annual Results Announcement\n",
              "1921  https://www1.hkexnews.hk/listedco/listconews/s...  ...                     Annual Results Announcement\n",
              "81    https://www1.hkexnews.hk/listedco/listconews/g...  ...                     Annual Results Announcement\n",
              "2153  https://www1.hkexnews.hk/listedco/listconews/s...  ...                     Annual Results Announcement\n",
              "\n",
              "[11933 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OPedAMQt8NE8"
      },
      "source": [
        "# Shuffle dataframe\n",
        "df = df.sample(frac = 1)"
      ],
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 698
        },
        "id": "TkD34Gm68R6t",
        "outputId": "6f2b3e46-b4a4-4854-c3af-5b7953708db6"
      },
      "source": [
        "# Create matrix of labels, precursor to one-hot encodings\n",
        "df['Notifiable Transaction'] = df.Label.apply(lambda x: 1 if x == \"Notifiable Transactions\" else 0)\n",
        "df['Takeover Offer'] = df.Label.apply(lambda x: 1 if x == \"Takeovers Code 3.7 Announcement\" else 0)\n",
        "df['Connected Transaction'] = df.Label.apply(lambda x:1 if x == \"Chapter 14A Connected Transaction Announcement\" else 0)\n",
        "df['Notice of AGM'] = df.Label.apply(lambda x:1 if x == \"Notice of AGM\" else 0)\n",
        "df['Trading Halt'] = df.Label.apply(lambda x:1 if x == 'Trading Halt' else 0)\n",
        "df['Annual Results'] = df.Label.apply(lambda x:1 if x == 'Annual Results Announcement' else 0)\n",
        "df.head()"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>URL</th>\n",
              "      <th>Text</th>\n",
              "      <th>Label</th>\n",
              "      <th>Notifiable Transaction</th>\n",
              "      <th>Takeover Offer</th>\n",
              "      <th>Connected Transaction</th>\n",
              "      <th>Notice of AGM</th>\n",
              "      <th>Trading Halt</th>\n",
              "      <th>Annual Results</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7441</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Notice of AGM</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2064</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Annual Results Announcement</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1791</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Annual Results Announcement</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong  Kong  Exchanges  and  Clearing  Limited ...</td>\n",
              "      <td>Takeovers Code 3.7 Announcement</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2350</th>\n",
              "      <td>https://www1.hkexnews.hk/listedco/listconews/s...</td>\n",
              "      <td>Hong Kong Exchanges and Clearing Limited and T...</td>\n",
              "      <td>Notifiable Transactions</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    URL  ... Annual Results\n",
              "7441  https://www1.hkexnews.hk/listedco/listconews/s...  ...              0\n",
              "2064  https://www1.hkexnews.hk/listedco/listconews/s...  ...              1\n",
              "1791  https://www1.hkexnews.hk/listedco/listconews/s...  ...              1\n",
              "202   https://www1.hkexnews.hk/listedco/listconews/s...  ...              0\n",
              "2350  https://www1.hkexnews.hk/listedco/listconews/s...  ...              0\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        },
        "id": "Jx9omPg7Bj7R",
        "outputId": "e7e77020-e31a-4f7b-a59b-1e360452e5cf"
      },
      "source": [
        "# Extract matrix of labels\n",
        "labels = df[['Notifiable Transaction', 'Takeover Offer', 'Connected Transaction', 'Notice of AGM', 'Trading Halt', 'Annual Results']]\n",
        "labels.head()"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Notifiable Transaction</th>\n",
              "      <th>Takeover Offer</th>\n",
              "      <th>Connected Transaction</th>\n",
              "      <th>Notice of AGM</th>\n",
              "      <th>Trading Halt</th>\n",
              "      <th>Annual Results</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7441</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2064</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1791</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2350</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      Notifiable Transaction  Takeover Offer  ...  Trading Halt  Annual Results\n",
              "7441                       0               0  ...             0               0\n",
              "2064                       0               0  ...             0               1\n",
              "1791                       0               0  ...             0               1\n",
              "202                        0               1  ...             0               0\n",
              "2350                       1               0  ...             0               0\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QLENC2zPXeTH"
      },
      "source": [
        "## Text Preprocessing\n",
        "\n",
        "Text preprocessing is an important step of any NLP application. My preliminary testing on the model with fewer labels and fewer training examples indicated that removing stop words leads to a drastically decreased accuracy on the classifier. This may be because a bidirectional LSTM would be more able to capture the semantic meanings of the corpus with the stop words intact. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B63v_rp-hGNY"
      },
      "source": [
        "# Text Preprocessing, removing stop words vastly decreased training accuracy\n",
        "def preprocess(corpus):\n",
        "    corpus = corpus.lower()\n",
        "    cleaner = re.compile(r'[^a-z 0-9]')\n",
        "    corpus = re.sub('\\n', ' ', corpus)\n",
        "    corpus = re.sub(cleaner, ' ', corpus)\n",
        "    corpus = re.sub(r'\\s+', ' ', corpus)\n",
        "  #  corpus = ' '.join([word for word in corpus.split() if word not in stopwords.words('english')])\n",
        "    return corpus"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZgPSS8g3CzNj"
      },
      "source": [
        "# Extract X Y values\n",
        "corpus_list = list(df.Text)\n",
        "X = [preprocess(corpus) for corpus in corpus_list]\n",
        "\n",
        "Y = labels.values"
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tXIHQLU6ATOL"
      },
      "source": [
        "# Create train-test split\n",
        "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wDdlG3icLBBO"
      },
      "source": [
        "# Tokenization and Padding\n",
        "\n",
        "Each text corpus in the training examples X_train are converted into an array of tokens, where each token will be word-vectors representating the token. \n",
        "\n",
        "Each of the inputs will be normalized in terms of length, with a max length on each input chosen to be 1500. This number came about after extensive testing, in addition to some intuitive feature engineering, such as knowing that the most important content of any announcement is located towards the beginning of the announcement. \n",
        "\n",
        "For announcements with less than 1500 words, null tokens will be used to pad the announcement until the 1500 word count is reached.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qcru1ir9FAC1",
        "outputId": "5627a36d-1ff5-4fe4-907d-f44863596040"
      },
      "source": [
        "# Process X data into tokenized and padded input\n",
        "tokenizer = Tokenizer(num_words=5000)\n",
        "tokenizer.fit_on_texts(X_train)\n",
        "\n",
        "X_train = tokenizer.texts_to_sequences(X_train)\n",
        "X_test = tokenizer.texts_to_sequences(X_test)\n",
        "\n",
        "vocab_size = len(tokenizer.word_index) + 1\n",
        "max_len = 3000\n",
        "X_train = pad_sequences(X_train, padding = 'post', maxlen = max_len)\n",
        "X_test = pad_sequences(X_test, padding = 'post', maxlen = max_len)\n",
        "\n",
        "print(f'vocab_size = {vocab_size}')"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "vocab_size = 74794\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dzuFkzGN2m1a",
        "outputId": "4f6bf1c9-6db1-475a-df31-f1a7216361c5"
      },
      "source": [
        "import keras\n",
        "(x_train, y_train), (x_val, y_val) = keras.datasets.imdb.load_data(num_words=vocab_size)\n"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<string>:6: VisibleDeprecationWarning:\n",
            "\n",
            "Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/datasets/imdb.py:159: VisibleDeprecationWarning:\n",
            "\n",
            "Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/datasets/imdb.py:160: VisibleDeprecationWarning:\n",
            "\n",
            "Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KUyPz2aL3tky"
      },
      "source": [
        "x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_len)\n",
        "x_val = keras.preprocessing.sequence.pad_sequences(x_val, maxlen=max_len)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GirUv_hQ3xQQ",
        "outputId": "bf1b8cb1-6940-43ad-83cd-bef836187434"
      },
      "source": [
        "X_train.shape, x_train.shape"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((9546, 2000), (25000, 2000))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YXHJuhAqPQwW"
      },
      "source": [
        "class TransformerBlock(tf.keras.layers.Layer):\r\n",
        "    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\r\n",
        "        super(TransformerBlock, self).__init__()\r\n",
        "        self.att = MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\r\n",
        "        self.ffn = Sequential([Dense(ff_dim, activation='relu'), Dense(embed_dim),])\r\n",
        "        self.layernorm1 = LayerNormalization() \r\n",
        "        self.layernorm2 = LayerNormalization() \r\n",
        "        self.dropout1 = Dropout(rate)\r\n",
        "        self.dropout2 = Dropout(rate)\r\n",
        "\r\n",
        "    def call(self, inputs, training):\r\n",
        "        attn_output = self.att(inputs, inputs)\r\n",
        "        attn_output = self.dropout1(attn_output, training=training)\r\n",
        "        out1 = self.layernorm1(inputs + attn_output)\r\n",
        "        ffn_output = self.ffn(out1)\r\n",
        "        ffn_output = self.dropout2(ffn_output, training=training)\r\n",
        "        return self.layernorm2(out1 + ffn_output)\r\n"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TNVCJaQNRs8D"
      },
      "source": [
        "class TokenPositionEmbedding(tf.keras.layers.Layer):\r\n",
        "    def __init__(self, maxlen, vocab_size, embed_dim):\r\n",
        "        super(TokenPositionEmbedding, self).__init__()\r\n",
        "        self.token_emb = Embedding(input_dim=vocab_size, output_dim=embed_dim)\r\n",
        "        self.pos_emb = Embedding(input_dim=maxlen, output_dim=embed_dim)\r\n",
        "    \r\n",
        "    def call(self,x):\r\n",
        "        maxlen=tf.shape(x)[-1]\r\n",
        "        positions = tf.range(start=0, limit=maxlen, delta=1)\r\n",
        "        positions = self.pos_emb(positions)\r\n",
        "        x = self.token_emb(x)\r\n",
        "        return x + positions \r\n"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pSq4mJ-Ev9Kq"
      },
      "source": [
        "def Transformers():\n",
        "    inputs = Input(shape = (max_len,))\n",
        "    embedding_layer = TokenPositionEmbedding(max_len, vocab_size, embed_dim)\n",
        "    layer = embedding_layer(inputs)\n",
        "    transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\n",
        "    layer = transformer_block(layer)\n",
        "    layer = GlobalAveragePooling1D()(layer)\n",
        "    layer = Dropout(0.2)(layer)\n",
        "    layer = Dense(128, activation='relu')(layer)\n",
        "    layer = Dropout(0.2)(layer)\n",
        "    outputs = Dense(6, activation='softmax')(layer)\n",
        "    model = Model(inputs=inputs, outputs=outputs)\n",
        "    return model"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6KmJ82nOv9Gi"
      },
      "source": [
        "embed_dim = 32  # Embedding size for each token\n",
        "num_heads = 2  # Number of attention heads\n",
        "ff_dim = 32  # Hidden layer size in feed forward network inside transformer\n",
        "\n",
        "model = Transformers()\n",
        "opt = tf.keras.optimizers.Adam(lr=1e-3)\n",
        "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KWTbY8KNA2Kd",
        "outputId": "00e80f7a-bd03-46dc-a5b7-6660412d1e75"
      },
      "source": [
        "model.summary()\n",
        "#tf.keras.utils.plot_model(model, to_file='model.png', show_shapes=True, show_layer_names=False)"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_7\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_10 (InputLayer)        [(None, 3000)]            0         \n",
            "_________________________________________________________________\n",
            "token_position_embedding_8 ( (None, 3000, 32)          2489408   \n",
            "_________________________________________________________________\n",
            "transformer_block_8 (Transfo (None, 3000, 32)          10656     \n",
            "_________________________________________________________________\n",
            "global_average_pooling1d_8 ( (None, 32)                0         \n",
            "_________________________________________________________________\n",
            "dropout_34 (Dropout)         (None, 32)                0         \n",
            "_________________________________________________________________\n",
            "dense_33 (Dense)             (None, 128)               4224      \n",
            "_________________________________________________________________\n",
            "dropout_35 (Dropout)         (None, 128)               0         \n",
            "_________________________________________________________________\n",
            "dense_34 (Dense)             (None, 6)                 774       \n",
            "=================================================================\n",
            "Total params: 2,505,062\n",
            "Trainable params: 2,505,062\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "92X6CfQTDzan",
        "outputId": "09295d9e-5edf-4373-cf77-d35240d56cf5"
      },
      "source": [
        "history = model.fit(X_train, Y_train,\n",
        "                    epochs=10,\n",
        "                    validation_data=(X_test, Y_test),\n",
        "                    batch_size=32)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "299/299 [==============================] - 144s 479ms/step - loss: 0.9077 - accuracy: 0.6696 - val_loss: 0.2130 - val_accuracy: 0.9145\n",
            "Epoch 2/10\n",
            "299/299 [==============================] - 143s 478ms/step - loss: 0.2095 - accuracy: 0.9260 - val_loss: 0.1591 - val_accuracy: 0.9426\n",
            "Epoch 3/10\n",
            "299/299 [==============================] - 143s 478ms/step - loss: 0.1224 - accuracy: 0.9564 - val_loss: 0.1334 - val_accuracy: 0.9518\n",
            "Epoch 4/10\n",
            "299/299 [==============================] - 143s 478ms/step - loss: 0.0722 - accuracy: 0.9766 - val_loss: 0.1350 - val_accuracy: 0.9548\n",
            "Epoch 5/10\n",
            "299/299 [==============================] - 143s 480ms/step - loss: 0.0509 - accuracy: 0.9843 - val_loss: 0.1626 - val_accuracy: 0.9535\n",
            "Epoch 6/10\n",
            "299/299 [==============================] - 143s 478ms/step - loss: 0.0419 - accuracy: 0.9904 - val_loss: 0.1680 - val_accuracy: 0.9581\n",
            "Epoch 7/10\n",
            "299/299 [==============================] - 143s 479ms/step - loss: 0.0373 - accuracy: 0.9889 - val_loss: 0.2017 - val_accuracy: 0.9535\n",
            "Epoch 8/10\n",
            "299/299 [==============================] - 143s 478ms/step - loss: 0.0382 - accuracy: 0.9891 - val_loss: 0.1669 - val_accuracy: 0.9552\n",
            "Epoch 9/10\n",
            "299/299 [==============================] - 143s 478ms/step - loss: 0.0225 - accuracy: 0.9941 - val_loss: 0.1815 - val_accuracy: 0.9514\n",
            "Epoch 10/10\n",
            "299/299 [==============================] - 143s 478ms/step - loss: 0.0219 - accuracy: 0.9943 - val_loss: 0.1745 - val_accuracy: 0.9560\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pVPiSNR8H-_q",
        "outputId": "7ff43663-59f1-4a21-ffd5-473466353c9f"
      },
      "source": [
        "acc = model.evaluate(X_test, Y_test)\n",
        "print(f'Validation set results \\nLoss: {acc[0]:.4f} \\nAccuracy: {acc[1]:.4f}')"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "75/75 [==============================] - 10s 133ms/step - loss: 0.1745 - accuracy: 0.9560\n",
            "Validation set results \n",
            "Loss: 0.1745 \n",
            "Accuracy: 0.9560\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "id": "Wx5i4oYRD-xd",
        "outputId": "23258a09-1108-49a2-aa6e-4009294abae8"
      },
      "source": [
        "# Plot losses and accuracy with plotly\n",
        "h = history.history\n",
        "epochs = [i+1 for i in range(len(h['loss']))]\n",
        "cols = plotly.colors.DEFAULT_PLOTLY_COLORS\n",
        "\n",
        "fig = make_subplots(rows=1, cols=2, subplot_titles=('Training and Validation Accuracy', 'Training and Validation Loss'))\n",
        "\n",
        "fig.add_trace(go.Scatter(x=epochs, y=h['accuracy'], \n",
        "                         line=dict(width=1.5, color=cols[0]),\n",
        "                         name='Training Accuracy'), 1,1)\n",
        "fig.add_trace(go.Scatter(x=epochs, y=h['val_accuracy'], \n",
        "                         line=dict(width=1.5, color=cols[3]),\n",
        "                         name='Validation Accuracy'), 1,1)\n",
        "\n",
        "fig.add_trace(go.Scatter(x=epochs, y=h['loss'], \n",
        "                         line=dict(width=1.5, color=cols[0]),\n",
        "                         name='Training Loss'), 1,2)\n",
        "fig.add_trace(go.Scatter(x=epochs, y=h['val_loss'], \n",
        "                         line=dict(width=1.5, color=cols[3]),\n",
        "                         name='Validation Loss'), 1,2)\n",
        "\n",
        "fig.update_layout(height = 400, width=1000)\n",
        "fig.update_xaxes(title_text='Epochs')\n",
        "fig.update_yaxes(title_text='Accuracy', row=1,col=1)\n",
        "fig.update_yaxes(title_text='Loss', row=1,col=2)\n",
        "\n",
        "fig.show()"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"c6b9f54d-6668-4e78-8a5e-2c1e1c2fb708\" class=\"plotly-graph-div\" style=\"height:400px; width:1000px;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"c6b9f54d-6668-4e78-8a5e-2c1e1c2fb708\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        'c6b9f54d-6668-4e78-8a5e-2c1e1c2fb708',\n",
              "                        [{\"line\": {\"color\": \"rgb(31, 119, 180)\", \"width\": 1.5}, \"name\": \"Training Accuracy\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], \"xaxis\": \"x\", \"y\": [0.8015922904014587, 0.9329562187194824, 0.9590404629707336, 0.9736015200614929, 0.9849151372909546, 0.9892101287841797, 0.9892101287841797, 0.9898386597633362, 0.9925623536109924, 0.9931908845901489], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"rgb(214, 39, 40)\", \"width\": 1.5}, \"name\": \"Validation Accuracy\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], \"xaxis\": \"x\", \"y\": [0.9145370721817017, 0.9426057934761047, 0.9518224000930786, 0.9547549486160278, 0.953498125076294, 0.9581063985824585, 0.953498125076294, 0.9551738500595093, 0.9514034390449524, 0.9560117125511169], \"yaxis\": \"y\"}, {\"line\": {\"color\": \"rgb(31, 119, 180)\", \"width\": 1.5}, \"name\": \"Training Loss\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], \"xaxis\": \"x2\", \"y\": [0.5556433796882629, 0.18206962943077087, 0.11582248657941818, 0.0785890743136406, 0.05009907856583595, 0.03952091187238693, 0.03746664896607399, 0.036123041063547134, 0.027716178447008133, 0.026746073737740517], \"yaxis\": \"y2\"}, {\"line\": {\"color\": \"rgb(214, 39, 40)\", \"width\": 1.5}, \"name\": \"Validation Loss\", \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], \"xaxis\": \"x2\", \"y\": [0.21299625933170319, 0.15914443135261536, 0.13336345553398132, 0.13496576249599457, 0.1626034677028656, 0.16798453032970428, 0.20171718299388885, 0.16685034334659576, 0.18146473169326782, 0.17447137832641602], \"yaxis\": \"y2\"}],\n",
              "                        {\"annotations\": [{\"font\": {\"size\": 16}, \"showarrow\": false, \"text\": \"Training and Validation Accuracy\", \"x\": 0.225, \"xanchor\": \"center\", \"xref\": \"paper\", \"y\": 1.0, \"yanchor\": \"bottom\", \"yref\": \"paper\"}, {\"font\": {\"size\": 16}, \"showarrow\": false, \"text\": \"Training and Validation Loss\", \"x\": 0.775, \"xanchor\": \"center\", \"xref\": \"paper\", \"y\": 1.0, \"yanchor\": \"bottom\", \"yref\": \"paper\"}], \"height\": 400, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"width\": 1000, \"xaxis\": {\"anchor\": \"y\", \"domain\": [0.0, 0.45], \"title\": {\"text\": \"Epochs\"}}, \"xaxis2\": {\"anchor\": \"y2\", \"domain\": [0.55, 1.0], \"title\": {\"text\": \"Epochs\"}}, \"yaxis\": {\"anchor\": \"x\", \"domain\": [0.0, 1.0], \"title\": {\"text\": \"Accuracy\"}}, \"yaxis2\": {\"anchor\": \"x2\", \"domain\": [0.0, 1.0], \"title\": {\"text\": \"Loss\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('c6b9f54d-6668-4e78-8a5e-2c1e1c2fb708');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "yavJ6tXWc9F7"
      },
      "source": [
        "model.save('/content/drive/My Drive/Data/HKEX/transformers_model.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HFL-XFotX4Ak"
      },
      "source": [
        "## Testing on New Announcements \n",
        "\n",
        "Let's see how our classifier does on new data that it has not seen! \n",
        "\n",
        "I picked some recent announcements for testing. \n",
        "\n",
        "For the testing, it is important that the testing announcements are not in the training set or validation set. The classify_pdf function written below takes this into account.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FPoiD8zUTQCf",
        "outputId": "53300b89-53a9-4d4c-f505-7ead63c72ca8"
      },
      "source": [
        "import requests\n",
        "!pip install PyPDF2 \n",
        "import PyPDF2\n",
        "import io\n",
        "import warnings"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting PyPDF2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b4/01/68fcc0d43daf4c6bdbc6b33cc3f77bda531c86b174cac56ef0ffdb96faab/PyPDF2-1.26.0.tar.gz (77kB)\n",
            "\r\u001b[K     |████▎                           | 10kB 27.6MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 20kB 16.0MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 30kB 13.9MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 40kB 13.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 51kB 9.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 61kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 71kB 9.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 81kB 6.5MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: PyPDF2\n",
            "  Building wheel for PyPDF2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for PyPDF2: filename=PyPDF2-1.26.0-cp36-none-any.whl size=61087 sha256=86f9dd230bcc5d389c66d499d736e14717619ad65caffa6dc57efbc983f456f7\n",
            "  Stored in directory: /root/.cache/pip/wheels/53/84/19/35bc977c8bf5f0c23a8a011aa958acd4da4bbd7a229315c1b7\n",
            "Successfully built PyPDF2\n",
            "Installing collected packages: PyPDF2\n",
            "Successfully installed PyPDF2-1.26.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4OxXCpnpTmIB"
      },
      "source": [
        "def classify_pdf(url):\n",
        "    # if url for testing is in the existing data set, do not go ahead with testing\n",
        "    if len(df.loc[df['URL'] == url]) != 0:\n",
        "        return f'{url} in Training or Validation, try another url'\n",
        "    with warnings.catch_warnings():\n",
        "        warnings.simplefilter('ignore')\n",
        "        response = requests.get(url)\n",
        "        raw_data = response.content\n",
        "\n",
        "        pdf_content = io.BytesIO(raw_data)\n",
        "        pdf_reader = PyPDF2.PdfFileReader(pdf_content)\n",
        "\n",
        "        text_string = ''\n",
        "\n",
        "        if pdf_reader.isEncrypted:\n",
        "            pdf_reader.decrypt(\"\")\n",
        "\n",
        "        for i in range(pdf_reader.numPages):\n",
        "            text_string += pdf_reader.getPage(i).extractText()\n",
        "\n",
        "        text = [text_string]\n",
        "        seq = tokenizer.texts_to_sequences(text)\n",
        "        padded = pad_sequences(seq, maxlen=max_len)\n",
        "        pred = model.predict(padded)\n",
        "        labels_pred = ['Notifiable Transaction', 'Takeover Offer', 'Connected Transaction', 'Notice of AGM', 'Trading Halt', 'Annual Results']\n",
        "        normalised = pred/np.sum(pred)*100\n",
        "        confidence = float(normalised.astype(float).reshape(len(labels_pred),-1)[np.argmax(pred)])\n",
        "        print(f'Prediction: {labels_pred[np.argmax(pred)]} with {confidence:.2f}% confidence')"
      ],
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KyOR25BwUJR7",
        "outputId": "df675776-0404-4a79-cdf6-ccac75b34e1c"
      },
      "source": [
        "classify_pdf('https://www1.hkexnews.hk/listedco/listconews/sehk/2020/0925/2020092501332.pdf')\n"
      ],
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Prediction: Notifiable Transaction with 99.97% confidence\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GTTnPdQ3yxbY"
      },
      "source": [
        "Great! The classifier was able to correctly identify the announcement as a Notifiable Transaction, which is also known as a Discloseable Transaction. Let's try a few more announcements."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-3dNWRKZVmmz",
        "outputId": "fc440b7b-89c5-4c6e-e79b-9892863146d7"
      },
      "source": [
        "classify_pdf('https://www1.hkexnews.hk/listedco/listconews/sehk/2020/0925/2020092501350.pdf')"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Prediction: Annual Results with 99.99% confidence\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hg7vWSy70p1p"
      },
      "source": [
        "The classifier is very confident about this result, which turned out to be correct."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hzjm9ujjzeCY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "74507bdf-9d61-4073-b6cd-e5ad81128d34"
      },
      "source": [
        "classify_pdf('https://www1.hkexnews.hk/listedco/listconews/sehk/2020/0925/2020092501035.pdf')"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Prediction: Trading Halt with 100.00% confidence\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s0Ui1nC306FJ"
      },
      "source": [
        "As for this sample, the classifier is not too confident about it, but it turned out to be correct."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s3b8ARY_d0c4"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}