LSTM_classifier_Attn.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "40kNYCEFZdr2",
        "outputId": "81189735-051a-4b03-ae86-951e76edbe75"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Looking in links: https://download.pytorch.org/whl/torch_stable.html\n",
            "Collecting torch==1.8.0+cu111\n",
            "  Downloading https://download.pytorch.org/whl/cu111/torch-1.8.0%2Bcu111-cp38-cp38-linux_x86_64.whl (1982.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 GB\u001b[0m \u001b[31m875.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchvision==0.9.0+cu111\n",
            "  Downloading https://download.pytorch.org/whl/cu111/torchvision-0.9.0%2Bcu111-cp38-cp38-linux_x86_64.whl (17.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.6/17.6 MB\u001b[0m \u001b[31m57.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchaudio==0.8.0\n",
            "  Downloading torchaudio-0.8.0-cp38-cp38-manylinux1_x86_64.whl (1.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m37.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchtext==0.9.0\n",
            "  Downloading torchtext-0.9.0-cp38-cp38-manylinux1_x86_64.whl (7.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m93.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch==1.8.0+cu111) (4.5.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from torch==1.8.0+cu111) (1.22.4)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.8/dist-packages (from torchvision==0.9.0+cu111) (8.4.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from torchtext==0.9.0) (2.25.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from torchtext==0.9.0) (4.64.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->torchtext==0.9.0) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->torchtext==0.9.0) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->torchtext==0.9.0) (4.0.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->torchtext==0.9.0) (1.26.14)\n",
            "Installing collected packages: torch, torchvision, torchtext, torchaudio\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 1.13.1+cu116\n",
            "    Uninstalling torch-1.13.1+cu116:\n",
            "      Successfully uninstalled torch-1.13.1+cu116\n",
            "  Attempting uninstall: torchvision\n",
            "    Found existing installation: torchvision 0.14.1+cu116\n",
            "    Uninstalling torchvision-0.14.1+cu116:\n",
            "      Successfully uninstalled torchvision-0.14.1+cu116\n",
            "  Attempting uninstall: torchtext\n",
            "    Found existing installation: torchtext 0.14.1\n",
            "    Uninstalling torchtext-0.14.1:\n",
            "      Successfully uninstalled torchtext-0.14.1\n",
            "  Attempting uninstall: torchaudio\n",
            "    Found existing installation: torchaudio 0.13.1+cu116\n",
            "    Uninstalling torchaudio-0.13.1+cu116:\n",
            "      Successfully uninstalled torchaudio-0.13.1+cu116\n",
            "Successfully installed torch-1.8.0+cu111 torchaudio-0.8.0 torchtext-0.9.0 torchvision-0.9.0+cu111\n"
          ]
        }
      ],
      "source": [
        "!pip install torch==1.8.0+cu111 torchvision==0.9.0+cu111 torchaudio==0.8.0 torchtext==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/bhargaviparanjape/clickbait.git\n",
        "!gzip -d /content/clickbait/dataset/clickbait_data.gz\n",
        "!gzip -d /content/clickbait/dataset/non_clickbait_data.gz"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TAo3atcZe5G8",
        "outputId": "c2d9bec9-322b-41d4-ea08-3b3c272de63e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'clickbait'...\n",
            "remote: Enumerating objects: 104, done.\u001b[K\n",
            "remote: Total 104 (delta 0), reused 0 (delta 0), pack-reused 104\u001b[K\n",
            "Receiving objects: 100% (104/104), 1.54 MiB | 22.84 MiB/s, done.\n",
            "Resolving deltas: 100% (44/44), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "E3BGSZKbhq0u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f1 = open('/content/clickbait/dataset/clickbait_data', 'r')\n",
        "clickbait = f1.readlines()\n",
        "\n",
        "# clickbait\n",
        "clickbait = list(map(lambda x:x.strip(),clickbait))\n",
        "clickbait = list(filter(None, clickbait))\n",
        "\n",
        "cb_df = pd.DataFrame(clickbait, columns = ['text'])\n",
        "cb_df['label'] = 1\n",
        "\n",
        "f2 = open('/content/clickbait/dataset/non_clickbait_data', 'r')\n",
        "non_clickbait = f2.readlines()\n",
        "\n",
        "non_clickbait = list(map(lambda x:x.strip(),non_clickbait))\n",
        "non_clickbait = list(filter(None, non_clickbait))\n",
        "\n",
        "ncb_df = pd.DataFrame(non_clickbait, columns = ['text'])\n",
        "ncb_df['label'] = 0\n",
        "\n",
        "raw_df = pd.concat([cb_df, ncb_df], ignore_index=True)\n",
        "raw_df.to_csv('raw_data.csv')"
      ],
      "metadata": {
        "id": "BemVfqRIgQS8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lffFsqA8BbFV"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torchtext.legacy import data\n",
        "\n",
        "SEED = 1234\n",
        "\n",
        "torch.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "\n",
        "TEXT = data.Field(tokenize = 'spacy', tokenizer_language = 'en_core_web_sm')\n",
        "LABEL = data.LabelField(dtype = torch.float)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fields = [('Unnamed: 0', None), ('text', TEXT), (\"label\", LABEL)]\n",
        "\n",
        "raw_data = data.TabularDataset(path=\"raw_data.csv\",format=\"csv\",fields=fields,skip_header=True)"
      ],
      "metadata": {
        "id": "7XJp3kLrge8H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IatXWViqB3Pi",
        "outputId": "80546b90-fe75-48a8-b191-3fa55668c849"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of training examples: 25600\n",
            "Number of testing examples: 3200\n",
            "Number of testing examples: 3200\n"
          ]
        }
      ],
      "source": [
        "import random\n",
        "\n",
        "# train and validation splitting\n",
        "train_data,test_data = raw_data.split(split_ratio=0.80,random_state=random.seed(SEED))\n",
        "print(f'Number of training examples: {len(train_data)}')\n",
        "valid_data, test_data = test_data.split(split_ratio=0.50,random_state=random.seed(SEED))\n",
        "print(f'Number of testing examples: {len(valid_data)}')\n",
        "print(f'Number of testing examples: {len(test_data)}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xtSufWSGB_bw",
        "outputId": "5957c7e5-3698-4582-95d6-ada440abd31b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Unique tokens in TEXT vocabulary: 25002\n",
            "Unique tokens in LABEL vocabulary: 2\n",
            "[('You', 4507), ('\"', 4373), ('The', 4127), ('in', 3514), (',', 3273), ('to', 2716), ('To', 2590), (\"'s\", 2450), ('A', 2272), ('of', 2124), ('-', 2092), ('Your', 2074), ('Of', 1936), ('Are', 1713), ('In', 1692), ('Is', 1642), ('That', 1570), ('This', 1447), ('for', 1368), ('And', 1326)]\n",
            "['<unk>', '<pad>', 'You', '\"', 'The', 'in', ',', 'to', 'To', \"'s\"]\n",
            "defaultdict(None, {'1': 0, '0': 1})\n"
          ]
        }
      ],
      "source": [
        "MAX_VOCAB_SIZE = 25_000\n",
        "\n",
        "TEXT.build_vocab(train_data, max_size = MAX_VOCAB_SIZE)\n",
        "LABEL.build_vocab(train_data)\n",
        "print(f\"Unique tokens in TEXT vocabulary: {len(TEXT.vocab)}\")\n",
        "print(f\"Unique tokens in LABEL vocabulary: {len(LABEL.vocab)}\")\n",
        "print(TEXT.vocab.freqs.most_common(20))\n",
        "print(TEXT.vocab.itos[:10])\n",
        "print(LABEL.vocab.stoi)\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TEXT.vocab.stoi['<pad>']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uk9PDhdKoHRk",
        "outputId": "5b4c7315-1b09-4a57-a174-8e3394294390"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "s_fT0m_-T8Ew"
      },
      "outputs": [],
      "source": [
        "# Setting up mini batching using dataloaders and collate function\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "\n",
        "\n",
        "def encode_text_pipeline(source):\n",
        "  encoded_text = [TEXT.vocab[word] for word in source]\n",
        "  return torch.tensor(encoded_text, dtype = torch.int64)\n",
        "\n",
        "def encoded_label_pipeline(target):\n",
        "  encoded_label = LABEL.vocab[target]\n",
        "  return torch.tensor(encoded_label, dtype = torch.int64)\n",
        "\n",
        "def pad_function(batch):\n",
        "  x_data = []\n",
        "  y_data = []\n",
        "  x_lengths = []\n",
        "  for i in range(len(batch)):\n",
        "    x_lengths.append(len(batch[i].text))\n",
        "    encoded_text = encode_text_pipeline(batch[i].text)\n",
        "    x_data.append(encoded_text)\n",
        "    encoded_label = encoded_label_pipeline(batch[i].label)\n",
        "    y_data.append(encoded_label)\n",
        "  padded_data = pad_sequence(x_data, batch_first = True, padding_value=1)\n",
        "  return padded_data, torch.tensor(y_data), torch.tensor(x_lengths)\n",
        "\n",
        "\n",
        "def create_loader(dataset):\n",
        "  data_loader = DataLoader(dataset=dataset, batch_size=32, collate_fn = pad_function, drop_last = True)\n",
        "  return data_loader\n",
        "\n",
        "train_loader = create_loader(train_data)\n",
        "valid_loader = create_loader(valid_data)\n",
        "test_loader = create_loader(test_data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nI_gFLUXCEdY"
      },
      "outputs": [],
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn as nn\n",
        "class LSTM_classifier(nn.Module):\n",
        "    def __init__(self, input_dim, embedding_dim, hidden_dim1, output_dim):\n",
        "        super(LSTM_classifier, self).__init__()\n",
        "        self.n_layers = 1\n",
        "        self.hidden_dim = hidden_dim1\n",
        "        self.embedding = nn.Embedding(input_dim, embedding_dim)\n",
        "        self.lstm = nn.LSTM(embedding_dim, hidden_dim1, num_layers=2, dropout=0.2, batch_first = True)\n",
        "        self.attention = nn.MultiheadAttention(hidden_dim1, num_heads=1)\n",
        "        self.dense = nn.Linear(hidden_dim1, output_dim)\n",
        "\n",
        "    def encoder_block(self, lstm_output):\n",
        "      attn_output, attn_weights = self.attention(lstm_output, lstm_output, lstm_output)\n",
        "      dense_output = self.dense(torch.mean(attn_output, 0))\n",
        "      return dense_output\n",
        "\n",
        "    def forward(self, text, x_lengths):\n",
        "      batch_size = text.size(0)\n",
        "      embedded = self.embedding(text)\n",
        "      packed_embeddings = nn.utils.rnn.pack_padded_sequence(embedded, x_lengths, batch_first=True, enforce_sorted=False)\n",
        "      packed_output, (hidden, cell) = self.lstm(packed_embeddings)\n",
        "      output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n",
        "      prediction = self.encoder_block(output)\n",
        "      return prediction, hidden\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-lZ2obDVDmEl"
      },
      "outputs": [],
      "source": [
        "INPUT_DIM = len(TEXT.vocab)\n",
        "EMBEDDING_DIM = 256\n",
        "HIDDEN_DIM = 128\n",
        "OUTPUT_DIM = 1\n",
        "\n",
        "model = LSTM_classifier(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM ,OUTPUT_DIM)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eJ6_rLRuK2l7",
        "outputId": "4efd9ae7-b66d-4cc8-8dd9-6615c1e6c4a6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The model has 6,796,417 trainable parameters\n"
          ]
        }
      ],
      "source": [
        "def count_parameters(model):\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'The model has {count_parameters(model):,} trainable parameters')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IbufnJzvLAlf"
      },
      "outputs": [],
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "# optimizer = optim.SGD(model.parameters(), lr=1e-3)\n",
        "optimizer = optim.Adam(model.parameters(), lr = 1e-4 )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I1fHORT8LBOK"
      },
      "outputs": [],
      "source": [
        "criterion = nn.BCEWithLogitsLoss()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z3gdBHHULC7y"
      },
      "outputs": [],
      "source": [
        "model = model.to(device)\n",
        "criterion = criterion.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KF2EdETnLEwt"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import f1_score\n",
        "\n",
        "def binary_accuracy(preds, y):\n",
        "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
        "    correct = (rounded_preds == y).float()\n",
        "    acc = correct.sum() / len(correct)\n",
        "    f1 = f1_score(rounded_preds.tolist(), y.tolist())\n",
        "    return acc, f1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2o9l76H5LGm9"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm\n",
        "def train(model, iterator, optimizer, criterion):\n",
        "    model.train()\n",
        "    epoch_loss = []\n",
        "    epoch_accuracy = []\n",
        "    epoch_f1 = []\n",
        "    for batch_text, batch_labels, batch_lengths in tqdm(iterator):\n",
        "        batch_size = batch_text.size(0)\n",
        "        optimizer.zero_grad()\n",
        "        batch_labels = batch_labels.to(device)\n",
        "        batch_predictions, hidden = model(batch_text.to(device), batch_lengths)\n",
        "        hidden.detach()\n",
        "        batch_loss = criterion(batch_predictions.squeeze(), batch_labels.squeeze().float())\n",
        "        epoch_loss.append(batch_loss)\n",
        "        batch_loss.backward()\n",
        "        optimizer.step()\n",
        "        batch_acc, batch_f1 = binary_accuracy(batch_predictions.squeeze(), batch_labels.squeeze())\n",
        "        epoch_accuracy.append(batch_acc)\n",
        "        epoch_f1.append(batch_f1)\n",
        "    return sum(epoch_loss)/len(epoch_loss), sum(epoch_accuracy)/len(epoch_accuracy), sum(epoch_f1)/len(epoch_f1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bBwRhOrzLKSH"
      },
      "outputs": [],
      "source": [
        "def evaluate(model, iterator, criterion):\n",
        "    epoch_loss = []\n",
        "    epoch_accuracy = []\n",
        "    epoch_f1 = []\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for batch_text, batch_labels, batch_lengths in tqdm(iterator):\n",
        "            batch_size = batch_text.size(0)\n",
        "            batch_labels = batch_labels.to(device)\n",
        "            batch_predictions, hidden = model(batch_text.to(device), batch_lengths)\n",
        "            hidden.detach()\n",
        "            batch_loss = criterion(batch_predictions.squeeze(), batch_labels.squeeze().float())\n",
        "            epoch_loss.append(batch_loss)\n",
        "            batch_acc, batch_f1 = binary_accuracy(batch_predictions.squeeze(), batch_labels.squeeze())\n",
        "            epoch_accuracy.append(batch_acc)\n",
        "            epoch_f1.append(batch_f1)\n",
        "    return sum(epoch_loss)/len(epoch_loss), sum(epoch_accuracy)/len(epoch_accuracy), sum(epoch_f1)/len(epoch_f1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FOHkN5P5LMtl"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "\n",
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "63EPftxxLOxH",
        "outputId": "7346481e-41d3-4ad9-cc8a-2b8eaee5ffb7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch:  0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 800/800 [00:10<00:00, 74.60it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss:  tensor(0.1732, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "Accuracy:  tensor(0.9161, device='cuda:0')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 100/100 [00:00<00:00, 207.06it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss:  tensor(0.0811, device='cuda:0')\n",
            "Accuracy:  tensor(0.9719, device='cuda:0')\n",
            "Epoch: 01 | Epoch Time: 0m 11s\n",
            "Train Loss: tensor(0.1732, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "Train accuracy tensor(0.9161, device='cuda:0')\n",
            "Train F1 0.8734158539011839\n",
            "Val Loss: tensor(0.0811, device='cuda:0')\n",
            "Val accuracy tensor(0.9719, device='cuda:0')\n",
            "Val F1 0.9721097129110732\n",
            "Epoch:  1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 800/800 [00:10<00:00, 77.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss:  tensor(0.0457, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "Accuracy:  tensor(0.9837, device='cuda:0')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 100/100 [00:00<00:00, 181.69it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss:  tensor(0.0933, device='cuda:0')\n",
            "Accuracy:  tensor(0.9747, device='cuda:0')\n",
            "Epoch: 02 | Epoch Time: 0m 10s\n",
            "Train Loss: tensor(0.0457, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "Train accuracy tensor(0.9837, device='cuda:0')\n",
            "Train F1 0.9830967021228452\n",
            "Val Loss: tensor(0.0933, device='cuda:0')\n",
            "Val accuracy tensor(0.9747, device='cuda:0')\n",
            "Val F1 0.9746676402149371\n",
            "Epoch:  2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 800/800 [00:09<00:00, 81.30it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss:  tensor(0.0246, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "Accuracy:  tensor(0.9912, device='cuda:0')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 100/100 [00:00<00:00, 128.18it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss:  tensor(0.1403, device='cuda:0')\n",
            "Accuracy:  tensor(0.9734, device='cuda:0')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "N_EPOCHS = 10\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "tolerance = 2\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "    print('Epoch: ', epoch)\n",
        "    start_time = time.time()\n",
        "    train_epoch_loss, train_epoch_accuracy, train_epoch_F1 = train(model, train_loader, optimizer, criterion)\n",
        "    print('Loss: ', train_epoch_loss)\n",
        "    print('Accuracy: ', train_epoch_accuracy)\n",
        "    valid_epoch_loss, valid_epoch_accuracy, valid_epoch_F1 = evaluate(model, valid_loader, criterion)\n",
        "    print('Loss: ', valid_epoch_loss)\n",
        "    print('Accuracy: ', valid_epoch_accuracy)\n",
        "    end_time = time.time()\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    \n",
        "    if valid_epoch_loss < best_valid_loss:\n",
        "      best_valid_loss = valid_epoch_loss\n",
        "    else:\n",
        "      tolerance -= 1\n",
        "      if tolerance == 0:  \n",
        "        break\n",
        "      \n",
        "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
        "    print('Train Loss:', train_epoch_loss)\n",
        "    print('Train accuracy', train_epoch_accuracy)\n",
        "    print('Train F1', train_epoch_F1)\n",
        "    print('Val Loss:', valid_epoch_loss)\n",
        "    print('Val accuracy', valid_epoch_accuracy)\n",
        "    print('Val F1', valid_epoch_F1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s2vQBjeTLTTv",
        "outputId": "b812ed10-6032-4ceb-a2fb-f931a65bc9d7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 100/100 [00:00<00:00, 201.06it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test Loss tensor(0.1633, device='cuda:0')\n",
            "Test accuracy tensor(0.9691, device='cuda:0')\n",
            "Test F1 0.9680289578145098\n"
          ]
        }
      ],
      "source": [
        "# model.load_state_dict(torch.load('tut1-model.pt'))\n",
        "\n",
        "test_loss, test_acc, test_f1 = evaluate(model, test_loader, criterion)\n",
        "\n",
        "print('Test Loss',test_loss)\n",
        "print('Test accuracy', test_acc)\n",
        "print('Test F1', test_f1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DWT55qIWSJFu"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}