named_entity_recognition/NER_Fine_tune_BERT_with_Huggingface.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NER_Fine-tune_BERT_with_Huggingface.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rFvcIe4qz_2t"
      },
      "source": [
        "# Fine Tuning BERT for Named Entity Recognition"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k3s3-jK36NoK"
      },
      "source": [
        "## Adapted from https://github.com/abhimishra91/transformers-tutorials"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zt90X_Dw0B_T"
      },
      "source": [
        "Dataset can be downloaded from https://www.kaggle.com/abhinavwalia95/entity-annotated-corpus."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ADcVPC3FaqEh"
      },
      "source": [
        "<a id='section00'></a>\n",
        "### Installing packages for preparing the system\n",
        "\n",
        "We are installing 2 packages for the purposes of TPU execution and f1 metric score calculation respectively\n",
        "*You can skip this step if you already have these libraries installed in your environment*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pWbkd8Ld8MwL",
        "outputId": "032b45c3-5182-4f3a-d809-42e05cec1eb4"
      },
      "source": [
        "!curl -q https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n",
        "!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev\n",
        "!pip -q install seqeval\n",
        "! pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100  5116  100  5116    0     0  31386      0 --:--:-- --:--:-- --:--:-- 31386\n",
            "Updating... This may take around 2 minutes.\n",
            "Updating TPU runtime to pytorch-dev20200515 ...\n",
            "Found existing installation: torch 1.6.0a0+bf2bbd9\n",
            "Uninstalling torch-1.6.0a0+bf2bbd9:\n",
            "  Successfully uninstalled torch-1.6.0a0+bf2bbd9\n",
            "Found existing installation: torchvision 0.7.0a0+a6073f0\n",
            "Uninstalling torchvision-0.7.0a0+a6073f0:\n",
            "  Successfully uninstalled torchvision-0.7.0a0+a6073f0\n",
            "Copying gs://tpu-pytorch/wheels/torch-nightly+20200515-cp37-cp37m-linux_x86_64.whl...\n",
            "- [1 files][ 91.0 MiB/ 91.0 MiB]                                                \n",
            "Operation completed over 1 objects/91.0 MiB.                                     \n",
            "Copying gs://tpu-pytorch/wheels/torch_xla-nightly+20200515-cp37-cp37m-linux_x86_64.whl...\n",
            "- [1 files][119.5 MiB/119.5 MiB]                                                \n",
            "Operation completed over 1 objects/119.5 MiB.                                    \n",
            "Copying gs://tpu-pytorch/wheels/torchvision-nightly+20200515-cp37-cp37m-linux_x86_64.whl...\n",
            "/ [1 files][  2.3 MiB/  2.3 MiB]                                                \n",
            "Operation completed over 1 objects/2.3 MiB.                                      \n",
            "Processing ./torch-nightly+20200515-cp37-cp37m-linux_x86_64.whl\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from torch==nightly+20200515) (0.16.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch==nightly+20200515) (1.19.5)\n",
            "Installing collected packages: torch\n",
            "Done updating TPU runtime\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "fastai 1.0.61 requires torchvision, which is not installed.\n",
            "torchtext 0.10.0 requires torch==1.9.0, but you have torch 1.6.0a0+bf2bbd9 which is incompatible.\u001b[0m\n",
            "Successfully installed torch-1.6.0a0+bf2bbd9\n",
            "Processing ./torch_xla-nightly+20200515-cp37-cp37m-linux_x86_64.whl\n",
            "torch-xla is already installed with the same version as the provided wheel. Use --force-reinstall to force an installation of the wheel.\n",
            "Processing ./torchvision-nightly+20200515-cp37-cp37m-linux_x86_64.whl\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision==nightly+20200515) (1.19.5)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from torchvision==nightly+20200515) (1.6.0a0+bf2bbd9)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.7/dist-packages (from torchvision==nightly+20200515) (7.1.2)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from torch->torchvision==nightly+20200515) (0.16.0)\n",
            "Installing collected packages: torchvision\n",
            "Successfully installed torchvision-0.7.0a0+a6073f0\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "libomp5 is already the newest version (5.0.1-1).\n",
            "libopenblas-dev is already the newest version (0.2.20+ds-4).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 40 not upgraded.\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.9.2)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.4)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: huggingface-hub==0.0.12 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.12)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (5.4.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.5.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6ts5noAi0a6K"
      },
      "source": [
        "## Importing Python Libraries and preparing the environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NYUqKiOZdR1H"
      },
      "source": [
        "# Importing pytorch and the library for TPU execution\n",
        "\n",
        "import torch\n",
        "import torch_xla\n",
        "import torch_xla.core.xla_model as xm"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y3jTWir2cBlN"
      },
      "source": [
        "# Importing stock ml libraries\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import transformers\n",
        "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertForTokenClassification, BertTokenizer, BertConfig, BertModel\n",
        "\n",
        "# Preparing for TPU usage\n",
        "dev = xm.xla_device()"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8tyJXAmdaqEl"
      },
      "source": [
        "<a id='section02'></a>\n",
        "### Importing and Pre-Processing the domain data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PngfxuCZrLEM",
        "outputId": "5004ddc8-77d8-4843-be86-393317b3ae75"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/gdrive\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 220
        },
        "id": "81kDZbz2cDn7",
        "outputId": "8962f9b5-cbae-4b62-82f6-6a0c10fde490"
      },
      "source": [
        "data_path = '/content/gdrive/MyDrive/iTrustAI/ner_dataset/ner.csv'\n",
        "df = pd.read_csv(data_path, encoding = \"ISO-8859-1\", error_bad_lines=False)\n",
        "dataset=df.drop(['Unnamed: 0', 'lemma', 'next-lemma', 'next-next-lemma', 'next-next-pos',\n",
        "       'next-next-shape', 'next-next-word', 'next-pos', 'next-shape',\n",
        "       'next-word', 'prev-iob', 'prev-lemma', 'prev-pos',\n",
        "       'prev-prev-iob', 'prev-prev-lemma', 'prev-prev-pos', 'prev-prev-shape',\n",
        "       'prev-prev-word', 'prev-shape', 'prev-word','shape'],axis=1)\n",
        "dataset.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "b'Skipping line 281837: expected 25 fields, saw 34\\n'\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pos</th>\n",
              "      <th>sentence_idx</th>\n",
              "      <th>word</th>\n",
              "      <th>tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NNS</td>\n",
              "      <td>1.0</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>IN</td>\n",
              "      <td>1.0</td>\n",
              "      <td>of</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NNS</td>\n",
              "      <td>1.0</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>VBP</td>\n",
              "      <td>1.0</td>\n",
              "      <td>have</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>VBN</td>\n",
              "      <td>1.0</td>\n",
              "      <td>marched</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   pos  sentence_idx           word tag\n",
              "0  NNS           1.0      Thousands   O\n",
              "1   IN           1.0             of   O\n",
              "2  NNS           1.0  demonstrators   O\n",
              "3  VBP           1.0           have   O\n",
              "4  VBN           1.0        marched   O"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sdqhHeAqcLnO"
      },
      "source": [
        "# Creating a class to pull the words from the columns and create them into sentences\n",
        "\n",
        "class SentenceGetter(object):\n",
        "    \n",
        "    def __init__(self, dataset):\n",
        "        self.n_sent = 1\n",
        "        self.dataset = dataset\n",
        "        self.empty = False\n",
        "        agg_func = lambda s: [(w,p, t) for w,p, t in zip(s[\"word\"].values.tolist(),\n",
        "                                                       s['pos'].values.tolist(),\n",
        "                                                        s[\"tag\"].values.tolist())]\n",
        "        self.grouped = self.dataset.groupby(\"sentence_idx\").apply(agg_func)\n",
        "        self.sentences = [s for s in self.grouped]\n",
        "    \n",
        "    def get_next(self):\n",
        "        try:\n",
        "            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n",
        "            self.n_sent += 1\n",
        "            return s\n",
        "        except:\n",
        "            return None\n",
        "\n",
        "getter = SentenceGetter(dataset)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cqln3QJecNsJ"
      },
      "source": [
        "# Creating new lists and dicts that will be used at a later stage for reference and processing\n",
        "\n",
        "tags_vals = list(set(dataset[\"tag\"].values))\n",
        "tag2idx = {t: i for i, t in enumerate(tags_vals)}\n",
        "sentences = [' '.join([s[0] for s in sent]) for sent in getter.sentences]\n",
        "labels = [[s[2] for s in sent] for sent in getter.sentences]\n",
        "labels = [[tag2idx.get(l) for l in lab] for lab in labels]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ATMrCKRTaqEo"
      },
      "source": [
        "<a id='section03'></a>\n",
        "### Preparing the Dataset and Dataloader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kL0b1VIQcTAC"
      },
      "source": [
        "# Defining some key variables that will be used later on in the training\n",
        "\n",
        "MAX_LEN = 200\n",
        "TRAIN_BATCH_SIZE = 32\n",
        "VALID_BATCH_SIZE = 16\n",
        "EPOCHS = 5\n",
        "LEARNING_RATE = 2e-05\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-cased')"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IV72GFgq_ZYb"
      },
      "source": [
        "class CustomDataset(Dataset):\n",
        "    def __init__(self, tokenizer, sentences, labels, max_len):\n",
        "        self.len = len(sentences)\n",
        "        self.sentences = sentences\n",
        "        self.labels = labels\n",
        "        self.tokenizer = tokenizer\n",
        "        self.max_len = max_len\n",
        "        \n",
        "    def __getitem__(self, index):\n",
        "        sentence = str(self.sentences[index])\n",
        "        inputs = self.tokenizer.encode_plus(\n",
        "            sentence,\n",
        "            None,\n",
        "            add_special_tokens=True,\n",
        "            max_length=self.max_len,\n",
        "            pad_to_max_length=True,\n",
        "            return_token_type_ids=True\n",
        "        )\n",
        "        ids = inputs['input_ids']\n",
        "        mask = inputs['attention_mask']\n",
        "        label = self.labels[index]\n",
        "        label.extend([4]*200)\n",
        "        label=label[:200]\n",
        "\n",
        "        return {\n",
        "            'ids': torch.tensor(ids, dtype=torch.long),\n",
        "            'mask': torch.tensor(mask, dtype=torch.long),\n",
        "            'tags': torch.tensor(label, dtype=torch.long)\n",
        "        } \n",
        "    \n",
        "    def __len__(self):\n",
        "        return self.len"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7BvniiyvX-rB",
        "outputId": "dc22e0cc-5d5c-4a19-84ea-5b3d70d5697b"
      },
      "source": [
        "# Creating the dataset and dataloader for the neural network\n",
        "\n",
        "train_percent = 0.8\n",
        "train_size = int(train_percent*len(sentences))\n",
        "# train_dataset=df.sample(frac=train_size,random_state=200).reset_index(drop=True)\n",
        "# test_dataset=df.drop(train_dataset.index).reset_index(drop=True)\n",
        "train_sentences = sentences[0:train_size]\n",
        "train_labels = labels[0:train_size]\n",
        "\n",
        "test_sentences = sentences[train_size:]\n",
        "test_labels = labels[train_size:]\n",
        "\n",
        "print(\"FULL Dataset: {}\".format(len(sentences)))\n",
        "print(\"TRAIN Dataset: {}\".format(len(train_sentences)))\n",
        "print(\"TEST Dataset: {}\".format(len(test_sentences)))\n",
        "\n",
        "training_set = CustomDataset(tokenizer, train_sentences, train_labels, MAX_LEN)\n",
        "testing_set = CustomDataset(tokenizer, test_sentences, test_labels, MAX_LEN)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "FULL Dataset: 35177\n",
            "TRAIN Dataset: 28141\n",
            "TEST Dataset: 7036\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F05bou5zZYtV"
      },
      "source": [
        "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "test_params = {'batch_size': VALID_BATCH_SIZE,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "training_loader = DataLoader(training_set, **train_params)\n",
        "testing_loader = DataLoader(testing_set, **test_params)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5_6UKSYzaqEq"
      },
      "source": [
        "### Creating the Neural Network for Fine Tuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9vuIJrvSZble"
      },
      "source": [
        "# Creating the customized model, by adding a drop out and a dense layer on top of distil bert to get the final output for the model. \n",
        "\n",
        "class BERTClass(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(BERTClass, self).__init__()\n",
        "        self.l1 = transformers.BertForTokenClassification.from_pretrained('bert-base-cased', num_labels=18)\n",
        "        # self.l2 = torch.nn.Dropout(0.3)\n",
        "        # self.l3 = torch.nn.Linear(768, 200)\n",
        "    \n",
        "    def forward(self, ids, mask, labels):\n",
        "        output_1= self.l1(ids, mask, labels = labels)\n",
        "        # output_2 = self.l2(output_1[0])\n",
        "        # output = self.l3(output_2)\n",
        "        return output_1"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "CflOeT2-ZoV6",
        "outputId": "b9f71070-c4ae-41aa-d490-7fc78d7d0df2"
      },
      "source": [
        "model = BERTClass()\n",
        "model.to(dev)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BERTClass(\n",
              "  (l1): BertForTokenClassification(\n",
              "    (bert): BertModel(\n",
              "      (embeddings): BertEmbeddings(\n",
              "        (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
              "        (position_embeddings): Embedding(512, 768)\n",
              "        (token_type_embeddings): Embedding(2, 768)\n",
              "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (encoder): BertEncoder(\n",
              "        (layer): ModuleList(\n",
              "          (0): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (1): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (2): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (3): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (4): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (5): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (6): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (7): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (8): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (9): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (10): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (11): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (classifier): Linear(in_features=768, out_features=18, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DN_u9NC5aaa_"
      },
      "source": [
        "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ONN5vzIvaqEs"
      },
      "source": [
        "### Fine Tuning the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8aQ6WCk2a-Vd"
      },
      "source": [
        "def train(epoch):\n",
        "    model.train()\n",
        "    for _,data in enumerate(training_loader, 0):\n",
        "        ids = data['ids'].to(dev, dtype = torch.long)\n",
        "        mask = data['mask'].to(dev, dtype = torch.long)\n",
        "        targets = data['tags'].to(dev, dtype = torch.long)\n",
        "\n",
        "        loss = model(ids, mask, labels = targets)[0]\n",
        "\n",
        "        # optimizer.zero_grad()\n",
        "        if _%500==0:\n",
        "            print(f'Epoch: {epoch}, Loss:  {loss.item()}')\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        xm.optimizer_step(optimizer)\n",
        "        xm.mark_step() "
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "50oMGTe0bvl0",
        "outputId": "6c9f0846-ff83-48cc-e123-23ecc7952bc6"
      },
      "source": [
        "for epoch in range(1):\n",
        "    train(epoch)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2190: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 0, Loss:  2.7969329357147217\n",
            "Epoch: 0, Loss:  0.2794185280799866\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JsQ7AAFVaqEt"
      },
      "source": [
        "<a id='section06'></a>\n",
        "### Validating the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6OckC0XNkWWm"
      },
      "source": [
        "from seqeval.metrics import f1_score\n",
        "\n",
        "def flat_accuracy(preds, labels):\n",
        "    flat_preds = np.argmax(preds, axis=2).flatten()\n",
        "    flat_labels = labels.flatten()\n",
        "    return np.sum(flat_preds == flat_labels)/len(flat_labels)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9zwFzzBriLMO"
      },
      "source": [
        "def valid(model, testing_loader):\n",
        "    model.eval()\n",
        "    eval_loss = 0; eval_accuracy = 0\n",
        "    n_correct = 0; n_wrong = 0; total = 0\n",
        "    predictions , true_labels = [], []\n",
        "    nb_eval_steps, nb_eval_examples = 0, 0\n",
        "    with torch.no_grad():\n",
        "        for _, data in enumerate(testing_loader, 0):\n",
        "            ids = data['ids'].to(dev, dtype = torch.long)\n",
        "            mask = data['mask'].to(dev, dtype = torch.long)\n",
        "            targets = data['tags'].to(dev, dtype = torch.long)\n",
        "\n",
        "            output = model(ids, mask, labels=targets)\n",
        "            loss, logits = output[:2]\n",
        "            logits = logits.detach().cpu().numpy()\n",
        "            label_ids = targets.to('cpu').numpy()\n",
        "            predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n",
        "            true_labels.append(label_ids)\n",
        "            accuracy = flat_accuracy(logits, label_ids)\n",
        "            eval_loss += loss.mean().item()\n",
        "            eval_accuracy += accuracy\n",
        "            nb_eval_examples += ids.size(0)\n",
        "            nb_eval_steps += 1\n",
        "        eval_loss = eval_loss/nb_eval_steps\n",
        "        print(\"Validation loss: {}\".format(eval_loss))\n",
        "        print(\"Validation Accuracy: {}\".format(eval_accuracy/nb_eval_steps))"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "syP-YlzHaqEu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "38d40d87-a113-4408-ab33-ee1ecc8097f1"
      },
      "source": [
        "# To get the results on the validation set. This data is not seen by the model\n",
        "\n",
        "valid(model, testing_loader)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2190: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Validation loss: 0.16895917343314398\n",
            "Validation Accuracy: 0.37273555871212133\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}