cuda_bert-per_target.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c8c206d3-696c-4ca4-864c-549b21e00cdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import transformers\n",
    "from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from pylab import rcParams\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from collections import defaultdict\n",
    "from textwrap import wrap\n",
    "from tqdm import tqdm\n",
    "\n",
    "from stance_utils import *\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "\n",
    "sns.set(style='whitegrid', palette='muted', font_scale=1.2)\n",
    "HAPPY_COLORS_PALETTE = [\"#01BEFE\", \"#FFDD00\", \"#FF7D00\", \"#FF006D\", \"#ADFF02\", \"#8F00FF\"]\n",
    "\n",
    "sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))\n",
    "rcParams['figure.figsize'] = 12, 8\n",
    "\n",
    "RANDOM_SEED = 23\n",
    "np.random.seed(RANDOM_SEED)\n",
    "\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8abdf725",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e68af5eb-e441-4ff9-ba37-292d9c64f4a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdc86008-d2c3-4a74-a1c4-19a113421633",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4fe14356",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = {'FAVOR': np.array([1, 0, 0]), 'AGAINST': np.array([0, 1, 0]), 'NONE': np.array([0, 0, 1])}\n",
    "classes_ = np.array(['FAVOR', 'AGAINST', 'NONE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a5e07682",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_test(train_file, test_file, target):\n",
    "    \n",
    "    sentence_maxlen = 0\n",
    "    x_train = []\n",
    "    y_train = []\n",
    "\n",
    "    \n",
    "    with open(train_file, 'r') as trainfile:\n",
    "        for line in trainfile:\n",
    "            \n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "            \n",
    "            #if line[0].strip() != 'ID' and line[1].strip() == t:\n",
    "            if line[0].strip() != 'ID' and target in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                #tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_train.append(tweet)\n",
    "                y_train.append(classes[line[3].strip()])\n",
    "                \n",
    "#     with open('rich_data_feminism_athesim_abortion.txt','r') as newfile:\n",
    "#         for line in newfile:\n",
    "#             line = line.replace('#SemST', '').strip()\n",
    "#             line = line.split('\\t')\n",
    "#             tweet = line[0]\n",
    "#             tweet = process_tweet(tweet)\n",
    "#             if len(tweet) > sentence_maxlen:\n",
    "#                 sentence_maxlen = len(tweet)\n",
    "#             x_train.append(tweet)\n",
    "#             y_train.append(classes[line[1].strip()])\n",
    "\n",
    "    pp = len(x_train)\n",
    "    x_test = []\n",
    "    y_test = []\n",
    "    with open(test_file, 'r') as testfile:\n",
    "        for line in testfile:\n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "        \n",
    "\n",
    "            #if line[0] != 'ID' and line[1] == t:\n",
    "            if line[0] != 'ID' and target in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                #tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_test.append(tweet)\n",
    "                y_test.append(classes[line[3].strip()])\n",
    "\n",
    "\n",
    "    \n",
    "    return x_train, y_train, x_test, y_test, sentence_maxlen,pp\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b64fc9a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_file_m = '/data/parush/stance_mohammed/train.txt'\n",
    "test_data_file_m = '/data/parush/stance_mohammed/test.txt'\n",
    "TARGETS_m = [ 'Atheism','Climate Change is a Real Concern', 'Feminist Movement','Hillary Clinton', 'Legalization of Abortion' ]\n",
    "\n",
    "\n",
    "# train_data_file_s = '/data/parush/SomasundaranWiebe-politicalDebates/train.txt'\n",
    "# test_data_file_s = '/data/parush/SomasundaranWiebe-politicalDebates/test.txt'\n",
    "# TARGETS_s = ['god','healthcare','guns','gayRights','abortion', 'creation']\n",
    "\n",
    "\n",
    "# train_data_file_q = '/data/parush/Data_MPCHI/train.txt'\n",
    "# test_data_file_q = '/data/parush/Data_MPCHI/test.txt'\n",
    "# TARGETS_q = ['Are E-Cigarettes safe?','Does MMR Vaccine lead to autism in children?',\n",
    "#       'Does Sunlight exposure lead to skin cancer?','Does Vitamin C prevent common cold?',\n",
    "#       'Should women take HRT post-menopause?']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "65432643-a7a6-4c9f-b844-51793807d62d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, dataframe, tokenizer, max_len):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.data = dataframe\n",
    "        self.title = dataframe['TITLE']\n",
    "        self.targets = self.data.target_list\n",
    "        self.max_len = max_len\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.title)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        title = str(self.title[index])\n",
    "        title = \" \".join(title.split())\n",
    "\n",
    "        inputs = self.tokenizer.encode_plus(\n",
    "            title,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            max_length=self.max_len,\n",
    "            padding='max_length',\n",
    "            return_token_type_ids=True,\n",
    "            truncation=True\n",
    "        )\n",
    "        ids = inputs['input_ids']\n",
    "        mask = inputs['attention_mask']\n",
    "        token_type_ids = inputs[\"token_type_ids\"]\n",
    "\n",
    "\n",
    "        return {\n",
    "            'ids': torch.tensor(ids, dtype=torch.long),\n",
    "            'mask': torch.tensor(mask, dtype=torch.long),\n",
    "            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
    "            'targets': torch.tensor(self.targets[index], dtype=torch.float)\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f5fb37dc-3971-4689-b1bc-45a88c776ab3",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BERTClass(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BERTClass, self).__init__()\n",
    "        self.l1 = transformers.BertModel.from_pretrained('bert-base-uncased')\n",
    "        self.l2 = torch.nn.Dropout(0.3)\n",
    "        self.l3 = torch.nn.Linear(768, 3)\n",
    "    \n",
    "    def forward(self, ids, mask, token_type_ids):\n",
    "        _, output_1= self.l1(ids, attention_mask = mask, token_type_ids = token_type_ids,return_dict=False)\n",
    "        output_2 = self.l2(output_1)\n",
    "        output = self.l3(output_2)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "706dccf1-47a6-4fbd-a592-901c2aa19340",
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(outputs, targets):\n",
    "    return torch.nn.BCEWithLogitsLoss()(outputs, targets)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "23916d69-216e-40e8-927d-86da89bab9a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_ckp(checkpoint_fpath, model, optimizer):\n",
    "    \"\"\"\n",
    "    checkpoint_path: path to save checkpoint\n",
    "    model: model that we want to load checkpoint parameters into       \n",
    "    optimizer: optimizer we defined in previous training\n",
    "    \"\"\"\n",
    "    # load check point\n",
    "    checkpoint = torch.load(checkpoint_fpath)\n",
    "    # initialize state_dict from checkpoint to model\n",
    "    model.load_state_dict(checkpoint['state_dict'])\n",
    "    # initialize optimizer from checkpoint to optimizer\n",
    "    optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "    # initialize valid_loss_min from checkpoint to valid_loss_min\n",
    "    valid_loss_min = checkpoint['valid_loss_min']\n",
    "    # return model, optimizer, epoch value, min validation loss \n",
    "    return model, optimizer, checkpoint['epoch'], valid_loss_min.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4b246446-4d2e-4a49-adf4-36e93df402cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil, sys   \n",
    "def save_ckp(state, is_best, checkpoint_path, best_model_path):\n",
    "    \"\"\"\n",
    "    state: checkpoint we want to save\n",
    "    is_best: is this the best checkpoint; min validation loss\n",
    "    checkpoint_path: path to save checkpoint\n",
    "    best_model_path: path to save best model\n",
    "    \"\"\"\n",
    "    f_path = checkpoint_path\n",
    "    # save checkpoint data to the path given, checkpoint_path\n",
    "    torch.save(state, f_path)\n",
    "    # if it is a best model, min validation loss\n",
    "    if is_best:\n",
    "        best_fpath = best_model_path\n",
    "        # copy that checkpoint file to best path given, best_model_path\n",
    "        shutil.copyfile(f_path, best_fpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "81d0e7bf-0571-446e-b6a8-3565ce5a5a3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(start_epochs,  n_epochs, valid_loss_min_input, \n",
    "                training_loader, validation_loader, model, \n",
    "                optimizer, checkpoint_path, best_model_path):\n",
    "   \n",
    "  # initialize tracker for minimum validation loss\n",
    "    valid_loss_min = valid_loss_min_input \n",
    "    for epoch in range(start_epochs, n_epochs+1):\n",
    "        train_loss = 0\n",
    "        valid_loss = 0\n",
    "\n",
    "        model.train()\n",
    "        print('############# Epoch {}: Training Start   #############'.format(epoch))\n",
    "        for batch_idx, data in enumerate(training_loader):\n",
    "            #print('yyy epoch', batch_idx)\n",
    "            ids = data['ids'].to(device, dtype = torch.long)\n",
    "            mask = data['mask'].to(device, dtype = torch.long)\n",
    "            token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "            targets = data['targets'].to(device, dtype = torch.float)\n",
    "\n",
    "            outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss = loss_fn(outputs, targets)\n",
    "            #if batch_idx%5000==0:\n",
    "             #   print(f'Epoch: {epoch}, Training Loss:  {loss.item()}')\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            #print('before loss data in training', loss.item(), train_loss)\n",
    "            train_loss = train_loss + ((1 / (batch_idx + 1)) * (loss.item() - train_loss))\n",
    "            #print('after loss data in training', loss.item(), train_loss)\n",
    "\n",
    "        print('############# Epoch {}: Training End     #############'.format(epoch))\n",
    "\n",
    "        print('############# Epoch {}: Validation Start   #############'.format(epoch))\n",
    "        ######################    \n",
    "        # validate the model #\n",
    "        ######################\n",
    "\n",
    "        model.eval()\n",
    "        val_targets, val_outputs = [] , []\n",
    "        with torch.no_grad():\n",
    "            for batch_idx, data in enumerate(validation_loader, 0):\n",
    "                ids = data['ids'].to(device, dtype = torch.long)\n",
    "                mask = data['mask'].to(device, dtype = torch.long)\n",
    "                token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "                targets = data['targets'].to(device, dtype = torch.float)\n",
    "                outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "                loss = loss_fn(outputs, targets)\n",
    "                valid_loss = valid_loss + ((1 / (batch_idx + 1)) * (loss.item() - valid_loss))                \n",
    "                \n",
    "                val_targets.extend(targets.cpu().detach().numpy().tolist())\n",
    "                val_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())\n",
    "            print('############# Epoch {}: Validation End     #############'.format(epoch))\n",
    "            # calculate average losses\n",
    "            #print('before cal avg train loss', train_loss)\n",
    "            train_loss = train_loss/len(training_loader)\n",
    "            valid_loss = valid_loss/len(validation_loader)\n",
    "            # print training/validation statistics \n",
    "            print('Epoch: {} \\tAvgerage Training Loss: {:.6f} \\tAverage Validation Loss: {:.6f}'.format(\n",
    "                epoch, \n",
    "                train_loss,\n",
    "                valid_loss\n",
    "                ))\n",
    "            # create checkpoint variable and add important data\n",
    "            checkpoint = {\n",
    "                'epoch': epoch + 1,\n",
    "                'valid_loss_min': valid_loss,\n",
    "                'state_dict': model.state_dict(),\n",
    "                'optimizer': optimizer.state_dict()\n",
    "            }\n",
    "\n",
    "            # save checkpoint\n",
    "            save_ckp(checkpoint, False, checkpoint_path, best_model_path)\n",
    "            ## TODO: save the model if validation loss has decreased\n",
    "            if valid_loss <= valid_loss_min:\n",
    "                print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(valid_loss_min,valid_loss))\n",
    "                # save checkpoint as best model\n",
    "                save_ckp(checkpoint, True, checkpoint_path, best_model_path)\n",
    "                valid_loss_min = valid_loss\n",
    "\n",
    "        print('############# Epoch {}  Done   #############\\n'.format(epoch))\n",
    "    \n",
    "    return model,val_targets, val_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "13696437",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FULL Dataset: (513, 2)\n",
      "TRAIN Dataset: (410, 2)\n",
      "TEST Dataset: (103, 2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA error: out of memory\nCUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_31608/575132110.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0mvalidation_loader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mtest_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBERTClass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m     \u001b[0moptimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mLEARNING_RATE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0mcheckpoint_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'/home/p/parush/bert_data/current_checkpoint_abortion.pt'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/nlp/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mto\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    850\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_floating_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_complex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 852\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    853\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    854\u001b[0m     def register_backward_hook(\n",
      "\u001b[0;32m~/.conda/envs/nlp/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    528\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/nlp/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    528\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/nlp/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    528\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/nlp/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    550\u001b[0m                 \u001b[0;31m# `with torch.no_grad():`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    551\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 552\u001b[0;31m                     \u001b[0mparam_applied\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    553\u001b[0m                 \u001b[0mshould_use_set_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    554\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mshould_use_set_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/nlp/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mconvert\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    848\u001b[0m                 return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None,\n\u001b[1;32m    849\u001b[0m                             non_blocking, memory_format=convert_to_format)\n\u001b[0;32m--> 850\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_floating_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_complex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    852\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: out of memory\nCUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1."
     ]
    }
   ],
   "source": [
    "# In[273]:\n",
    "for target in TARGETS_m:\n",
    "    \n",
    "    MAX_LEN = 128\n",
    "    TRAIN_BATCH_SIZE = 32\n",
    "    VALID_BATCH_SIZE = 32\n",
    "    EPOCHS = 20\n",
    "    LEARNING_RATE = 1e-05\n",
    "    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
    "    train_texts, train_labels, test_texts, test_labels, sen_maxlen, pp = train_and_test(train_data_file_m, test_data_file_m, target)\n",
    "    df2 = pd.DataFrame([])\n",
    "    df2['TITLE'] = train_texts\n",
    "    df2['target_list'] = train_labels\n",
    "    df_test = pd.DataFrame([])\n",
    "    df_test['TITLE'] = test_texts\n",
    "    df_test['target_list'] = test_labels\n",
    "    train_size = 0.8\n",
    "    train_dataset = df2.sample(frac=train_size,random_state=200)\n",
    "    valid_dataset = df2.drop(train_dataset.index).reset_index(drop=True)\n",
    "    train_dataset = train_dataset.reset_index(drop=True)\n",
    "\n",
    "\n",
    "    print(\"FULL Dataset: {}\".format(df2.shape))\n",
    "    print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
    "    print(\"TEST Dataset: {}\".format(valid_dataset.shape))\n",
    "    \n",
    "    training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)\n",
    "    validation_set = CustomDataset(valid_dataset,  tokenizer, MAX_LEN)\n",
    "    train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "    test_params = {'batch_size': VALID_BATCH_SIZE,\n",
    "                'shuffle': False,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "    training_loader = DataLoader(training_set, **train_params)\n",
    "    validation_loader = DataLoader(validation_set, **test_params)\n",
    "    model = BERTClass()\n",
    "    model.to(device)\n",
    "    optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)\n",
    "    checkpoint_path = '/home/p/parush/bert_data/current_checkpoint_abortion.pt'\n",
    "    best_model = '/home/p/parush/bert_data/best_model_trained_on_abortion.pt'\n",
    "    trained_model,val_targets, val_outputs = train_model(1, EPOCHS, np.Inf, training_loader, validation_loader, model, \n",
    "                      optimizer,checkpoint_path,best_model)\n",
    "    val_preds = (np.array(val_outputs) > 0.5).astype(int)\n",
    "    print('Trained and tested on: ', target)\n",
    "    print(classification_report(val_targets, val_preds,labels=[0,1],digits = 4))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b377ce52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df2 = pd.DataFrame([])\n",
    "# df2['TITLE'] = train_texts\n",
    "# df2['target_list'] = train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e1732066-f056-4de7-88b6-de62fbba50c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_test = pd.DataFrame([])\n",
    "# df_test['TITLE'] = test_texts\n",
    "# df_test['target_list'] = test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ed968052-5a3f-4d54-8708-fa3ca0baff60",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6387baeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d415c695",
   "metadata": {},
   "outputs": [],
   "source": [
    "# MAX_LEN = 128\n",
    "# TRAIN_BATCH_SIZE = 32\n",
    "# VALID_BATCH_SIZE = 32\n",
    "# EPOCHS = 20\n",
    "# LEARNING_RATE = 1e-05\n",
    "# tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2572333c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8010ac26",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c312e63d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# class CustomDataset(Dataset):\n",
    "    \n",
    "#     def __init__(self, dataframe, tokenizer, max_len):\n",
    "#         self.tokenizer = tokenizer\n",
    "#         self.data = dataframe\n",
    "#         self.title = dataframe['TITLE']\n",
    "#         self.targets = self.data.target_list\n",
    "#         self.max_len = max_len\n",
    "\n",
    "#     def __len__(self):\n",
    "#         return len(self.title)\n",
    "\n",
    "#     def __getitem__(self, index):\n",
    "#         title = str(self.title[index])\n",
    "#         title = \" \".join(title.split())\n",
    "\n",
    "#         inputs = self.tokenizer.encode_plus(\n",
    "#             title,\n",
    "#             None,\n",
    "#             add_special_tokens=True,\n",
    "#             max_length=self.max_len,\n",
    "#             padding='max_length',\n",
    "#             return_token_type_ids=True,\n",
    "#             truncation=True\n",
    "#         )\n",
    "#         ids = inputs['input_ids']\n",
    "#         mask = inputs['attention_mask']\n",
    "#         token_type_ids = inputs[\"token_type_ids\"]\n",
    "\n",
    "\n",
    "#         return {\n",
    "#             'ids': torch.tensor(ids, dtype=torch.long),\n",
    "#             'mask': torch.tensor(mask, dtype=torch.long),\n",
    "#             'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
    "#             'targets': torch.tensor(self.targets[index], dtype=torch.float)\n",
    "#         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c8835f33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FULL Dataset: (653, 2)\n",
      "TRAIN Dataset: (522, 2)\n",
      "TEST Dataset: (131, 2)\n"
     ]
    }
   ],
   "source": [
    "# train_size = 0.8\n",
    "# train_dataset = df2.sample(frac=train_size,random_state=200)\n",
    "# valid_dataset = df2.drop(train_dataset.index).reset_index(drop=True)\n",
    "# train_dataset = train_dataset.reset_index(drop=True)\n",
    "\n",
    "\n",
    "# print(\"FULL Dataset: {}\".format(df2.shape))\n",
    "# print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
    "# print(\"TEST Dataset: {}\".format(valid_dataset.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e53f2f99-2038-4860-ad33-6292e1b12c6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)\n",
    "# validation_set = CustomDataset(valid_dataset,  tokenizer, MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d7b2fda",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "64a6bd3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
    "#                 'shuffle': True,\n",
    "#                 'num_workers': 0\n",
    "#                 }\n",
    "\n",
    "# test_params = {'batch_size': VALID_BATCH_SIZE,\n",
    "#                 'shuffle': False,\n",
    "#                 'num_workers': 0\n",
    "#                 }\n",
    "\n",
    "# training_loader = DataLoader(training_set, **train_params)\n",
    "# validation_loader = DataLoader(validation_set, **test_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0a2c231c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# class BERTClass(torch.nn.Module):\n",
    "#     def __init__(self):\n",
    "#         super(BERTClass, self).__init__()\n",
    "#         self.l1 = transformers.BertModel.from_pretrained('bert-base-uncased')\n",
    "#         self.l2 = torch.nn.Dropout(0.3)\n",
    "#         self.l3 = torch.nn.Linear(768, 3)\n",
    "    \n",
    "#     def forward(self, ids, mask, token_type_ids):\n",
    "#         _, output_1= self.l1(ids, attention_mask = mask, token_type_ids = token_type_ids,return_dict=False)\n",
    "#         output_2 = self.l2(output_1)\n",
    "#         output = self.l3(output_2)\n",
    "#         return output\n",
    "\n",
    "# model = BERTClass()\n",
    "# model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "79ae560b-82f7-4632-a2d0-c0ff037b2655",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def loss_fn(outputs, targets):\n",
    "#     return torch.nn.BCEWithLogitsLoss()(outputs, targets)\n",
    "\n",
    "# optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "31ee07de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def load_ckp(checkpoint_fpath, model, optimizer):\n",
    "#     \"\"\"\n",
    "#     checkpoint_path: path to save checkpoint\n",
    "#     model: model that we want to load checkpoint parameters into       \n",
    "#     optimizer: optimizer we defined in previous training\n",
    "#     \"\"\"\n",
    "#     # load check point\n",
    "#     checkpoint = torch.load(checkpoint_fpath)\n",
    "#     # initialize state_dict from checkpoint to model\n",
    "#     model.load_state_dict(checkpoint['state_dict'])\n",
    "#     # initialize optimizer from checkpoint to optimizer\n",
    "#     optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "#     # initialize valid_loss_min from checkpoint to valid_loss_min\n",
    "#     valid_loss_min = checkpoint['valid_loss_min']\n",
    "#     # return model, optimizer, epoch value, min validation loss \n",
    "#     return model, optimizer, checkpoint['epoch'], valid_loss_min.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f2b6d1a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import shutil, sys   \n",
    "# def save_ckp(state, is_best, checkpoint_path, best_model_path):\n",
    "#     \"\"\"\n",
    "#     state: checkpoint we want to save\n",
    "#     is_best: is this the best checkpoint; min validation loss\n",
    "#     checkpoint_path: path to save checkpoint\n",
    "#     best_model_path: path to save best model\n",
    "#     \"\"\"\n",
    "#     f_path = checkpoint_path\n",
    "#     # save checkpoint data to the path given, checkpoint_path\n",
    "#     torch.save(state, f_path)\n",
    "#     # if it is a best model, min validation loss\n",
    "#     if is_best:\n",
    "#         best_fpath = best_model_path\n",
    "#         # copy that checkpoint file to best path given, best_model_path\n",
    "#         shutil.copyfile(f_path, best_fpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6f1b0726",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def train_model(start_epochs,  n_epochs, valid_loss_min_input, \n",
    "#                 training_loader, validation_loader, model, \n",
    "#                 optimizer, checkpoint_path, best_model_path):\n",
    "   \n",
    "#   # initialize tracker for minimum validation loss\n",
    "#     valid_loss_min = valid_loss_min_input \n",
    "#     for epoch in range(start_epochs, n_epochs+1):\n",
    "#         train_loss = 0\n",
    "#         valid_loss = 0\n",
    "\n",
    "#         model.train()\n",
    "#         print('############# Epoch {}: Training Start   #############'.format(epoch))\n",
    "#         for batch_idx, data in enumerate(training_loader):\n",
    "#             #print('yyy epoch', batch_idx)\n",
    "#             ids = data['ids'].to(device, dtype = torch.long)\n",
    "#             mask = data['mask'].to(device, dtype = torch.long)\n",
    "#             token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "#             targets = data['targets'].to(device, dtype = torch.float)\n",
    "\n",
    "#             outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "#             optimizer.zero_grad()\n",
    "#             loss = loss_fn(outputs, targets)\n",
    "#             #if batch_idx%5000==0:\n",
    "#              #   print(f'Epoch: {epoch}, Training Loss:  {loss.item()}')\n",
    "\n",
    "#             optimizer.zero_grad()\n",
    "#             loss.backward()\n",
    "#             optimizer.step()\n",
    "#             #print('before loss data in training', loss.item(), train_loss)\n",
    "#             train_loss = train_loss + ((1 / (batch_idx + 1)) * (loss.item() - train_loss))\n",
    "#             #print('after loss data in training', loss.item(), train_loss)\n",
    "\n",
    "#         print('############# Epoch {}: Training End     #############'.format(epoch))\n",
    "\n",
    "#         print('############# Epoch {}: Validation Start   #############'.format(epoch))\n",
    "#         ######################    \n",
    "#         # validate the model #\n",
    "#         ######################\n",
    "\n",
    "#         model.eval()\n",
    "#         val_targets, val_outputs = [] , []\n",
    "#         with torch.no_grad():\n",
    "#             for batch_idx, data in enumerate(validation_loader, 0):\n",
    "#                 ids = data['ids'].to(device, dtype = torch.long)\n",
    "#                 mask = data['mask'].to(device, dtype = torch.long)\n",
    "#                 token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "#                 targets = data['targets'].to(device, dtype = torch.float)\n",
    "#                 outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "#                 loss = loss_fn(outputs, targets)\n",
    "#                 valid_loss = valid_loss + ((1 / (batch_idx + 1)) * (loss.item() - valid_loss))                \n",
    "                \n",
    "#                 val_targets.extend(targets.cpu().detach().numpy().tolist())\n",
    "#                 val_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())\n",
    "#             print('############# Epoch {}: Validation End     #############'.format(epoch))\n",
    "#             # calculate average losses\n",
    "#             #print('before cal avg train loss', train_loss)\n",
    "#             train_loss = train_loss/len(training_loader)\n",
    "#             valid_loss = valid_loss/len(validation_loader)\n",
    "#             # print training/validation statistics \n",
    "#             print('Epoch: {} \\tAvgerage Training Loss: {:.6f} \\tAverage Validation Loss: {:.6f}'.format(\n",
    "#                 epoch, \n",
    "#                 train_loss,\n",
    "#                 valid_loss\n",
    "#                 ))\n",
    "#             # create checkpoint variable and add important data\n",
    "#             checkpoint = {\n",
    "#                 'epoch': epoch + 1,\n",
    "#                 'valid_loss_min': valid_loss,\n",
    "#                 'state_dict': model.state_dict(),\n",
    "#                 'optimizer': optimizer.state_dict()\n",
    "#             }\n",
    "\n",
    "#             # save checkpoint\n",
    "#             save_ckp(checkpoint, False, checkpoint_path, best_model_path)\n",
    "#             ## TODO: save the model if validation loss has decreased\n",
    "#             if valid_loss <= valid_loss_min:\n",
    "#                 print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(valid_loss_min,valid_loss))\n",
    "#                 # save checkpoint as best model\n",
    "#                 save_ckp(checkpoint, True, checkpoint_path, best_model_path)\n",
    "#                 valid_loss_min = valid_loss\n",
    "\n",
    "#         print('############# Epoch {}  Done   #############\\n'.format(epoch))\n",
    "    \n",
    "#     return model,val_targets, val_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e49dcc84-7f6f-464d-a7b0-653877ff4053",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "id": "ba37a6c6-a36a-413e-9605-f77edf85ac39",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "85494eb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############# Epoch 1: Training Start   #############\n",
      "############# Epoch 1: Training End     #############\n",
      "############# Epoch 1: Validation Start   #############\n",
      "############# Epoch 1: Validation End     #############\n",
      "Epoch: 1 \tAvgerage Training Loss: 0.036592 \tAverage Validation Loss: 0.111198\n",
      "Validation loss decreased (inf --> 0.111198).  Saving model ...\n",
      "############# Epoch 1  Done   #############\n",
      "\n",
      "############# Epoch 2: Training Start   #############\n",
      "############# Epoch 2: Training End     #############\n",
      "############# Epoch 2: Validation Start   #############\n",
      "############# Epoch 2: Validation End     #############\n",
      "Epoch: 2 \tAvgerage Training Loss: 0.034328 \tAverage Validation Loss: 0.108167\n",
      "Validation loss decreased (0.111198 --> 0.108167).  Saving model ...\n",
      "############# Epoch 2  Done   #############\n",
      "\n",
      "############# Epoch 3: Training Start   #############\n",
      "############# Epoch 3: Training End     #############\n",
      "############# Epoch 3: Validation Start   #############\n",
      "############# Epoch 3: Validation End     #############\n",
      "Epoch: 3 \tAvgerage Training Loss: 0.033546 \tAverage Validation Loss: 0.104657\n",
      "Validation loss decreased (0.108167 --> 0.104657).  Saving model ...\n",
      "############# Epoch 3  Done   #############\n",
      "\n",
      "############# Epoch 4: Training Start   #############\n",
      "############# Epoch 4: Training End     #############\n",
      "############# Epoch 4: Validation Start   #############\n",
      "############# Epoch 4: Validation End     #############\n",
      "Epoch: 4 \tAvgerage Training Loss: 0.032487 \tAverage Validation Loss: 0.100113\n",
      "Validation loss decreased (0.104657 --> 0.100113).  Saving model ...\n",
      "############# Epoch 4  Done   #############\n",
      "\n",
      "############# Epoch 5: Training Start   #############\n",
      "############# Epoch 5: Training End     #############\n",
      "############# Epoch 5: Validation Start   #############\n",
      "############# Epoch 5: Validation End     #############\n",
      "Epoch: 5 \tAvgerage Training Loss: 0.030373 \tAverage Validation Loss: 0.095325\n",
      "Validation loss decreased (0.100113 --> 0.095325).  Saving model ...\n",
      "############# Epoch 5  Done   #############\n",
      "\n",
      "############# Epoch 6: Training Start   #############\n",
      "############# Epoch 6: Training End     #############\n",
      "############# Epoch 6: Validation Start   #############\n",
      "############# Epoch 6: Validation End     #############\n",
      "Epoch: 6 \tAvgerage Training Loss: 0.028006 \tAverage Validation Loss: 0.090170\n",
      "Validation loss decreased (0.095325 --> 0.090170).  Saving model ...\n",
      "############# Epoch 6  Done   #############\n",
      "\n",
      "############# Epoch 7: Training Start   #############\n",
      "############# Epoch 7: Training End     #############\n",
      "############# Epoch 7: Validation Start   #############\n",
      "############# Epoch 7: Validation End     #############\n",
      "Epoch: 7 \tAvgerage Training Loss: 0.025668 \tAverage Validation Loss: 0.086004\n",
      "Validation loss decreased (0.090170 --> 0.086004).  Saving model ...\n",
      "############# Epoch 7  Done   #############\n",
      "\n",
      "############# Epoch 8: Training Start   #############\n",
      "############# Epoch 8: Training End     #############\n",
      "############# Epoch 8: Validation Start   #############\n",
      "############# Epoch 8: Validation End     #############\n",
      "Epoch: 8 \tAvgerage Training Loss: 0.023010 \tAverage Validation Loss: 0.083166\n",
      "Validation loss decreased (0.086004 --> 0.083166).  Saving model ...\n",
      "############# Epoch 8  Done   #############\n",
      "\n",
      "############# Epoch 9: Training Start   #############\n",
      "############# Epoch 9: Training End     #############\n",
      "############# Epoch 9: Validation Start   #############\n",
      "############# Epoch 9: Validation End     #############\n",
      "Epoch: 9 \tAvgerage Training Loss: 0.021035 \tAverage Validation Loss: 0.079980\n",
      "Validation loss decreased (0.083166 --> 0.079980).  Saving model ...\n",
      "############# Epoch 9  Done   #############\n",
      "\n",
      "############# Epoch 10: Training Start   #############\n",
      "############# Epoch 10: Training End     #############\n",
      "############# Epoch 10: Validation Start   #############\n",
      "############# Epoch 10: Validation End     #############\n",
      "Epoch: 10 \tAvgerage Training Loss: 0.018584 \tAverage Validation Loss: 0.077072\n",
      "Validation loss decreased (0.079980 --> 0.077072).  Saving model ...\n",
      "############# Epoch 10  Done   #############\n",
      "\n",
      "############# Epoch 11: Training Start   #############\n",
      "############# Epoch 11: Training End     #############\n",
      "############# Epoch 11: Validation Start   #############\n",
      "############# Epoch 11: Validation End     #############\n",
      "Epoch: 11 \tAvgerage Training Loss: 0.016126 \tAverage Validation Loss: 0.075883\n",
      "Validation loss decreased (0.077072 --> 0.075883).  Saving model ...\n",
      "############# Epoch 11  Done   #############\n",
      "\n",
      "############# Epoch 12: Training Start   #############\n",
      "############# Epoch 12: Training End     #############\n",
      "############# Epoch 12: Validation Start   #############\n",
      "############# Epoch 12: Validation End     #############\n",
      "Epoch: 12 \tAvgerage Training Loss: 0.013953 \tAverage Validation Loss: 0.073603\n",
      "Validation loss decreased (0.075883 --> 0.073603).  Saving model ...\n",
      "############# Epoch 12  Done   #############\n",
      "\n",
      "############# Epoch 13: Training Start   #############\n",
      "############# Epoch 13: Training End     #############\n",
      "############# Epoch 13: Validation Start   #############\n",
      "############# Epoch 13: Validation End     #############\n",
      "Epoch: 13 \tAvgerage Training Loss: 0.011766 \tAverage Validation Loss: 0.070434\n",
      "Validation loss decreased (0.073603 --> 0.070434).  Saving model ...\n",
      "############# Epoch 13  Done   #############\n",
      "\n",
      "############# Epoch 14: Training Start   #############\n",
      "############# Epoch 14: Training End     #############\n",
      "############# Epoch 14: Validation Start   #############\n",
      "############# Epoch 14: Validation End     #############\n",
      "Epoch: 14 \tAvgerage Training Loss: 0.009853 \tAverage Validation Loss: 0.068179\n",
      "Validation loss decreased (0.070434 --> 0.068179).  Saving model ...\n",
      "############# Epoch 14  Done   #############\n",
      "\n",
      "############# Epoch 15: Training Start   #############\n",
      "############# Epoch 15: Training End     #############\n",
      "############# Epoch 15: Validation Start   #############\n",
      "############# Epoch 15: Validation End     #############\n",
      "Epoch: 15 \tAvgerage Training Loss: 0.007852 \tAverage Validation Loss: 0.071931\n",
      "############# Epoch 15  Done   #############\n",
      "\n",
      "############# Epoch 16: Training Start   #############\n",
      "############# Epoch 16: Training End     #############\n",
      "############# Epoch 16: Validation Start   #############\n",
      "############# Epoch 16: Validation End     #############\n",
      "Epoch: 16 \tAvgerage Training Loss: 0.006304 \tAverage Validation Loss: 0.070944\n",
      "############# Epoch 16  Done   #############\n",
      "\n",
      "############# Epoch 17: Training Start   #############\n",
      "############# Epoch 17: Training End     #############\n",
      "############# Epoch 17: Validation Start   #############\n",
      "############# Epoch 17: Validation End     #############\n",
      "Epoch: 17 \tAvgerage Training Loss: 0.004618 \tAverage Validation Loss: 0.070126\n",
      "############# Epoch 17  Done   #############\n",
      "\n",
      "############# Epoch 18: Training Start   #############\n",
      "############# Epoch 18: Training End     #############\n",
      "############# Epoch 18: Validation Start   #############\n",
      "############# Epoch 18: Validation End     #############\n",
      "Epoch: 18 \tAvgerage Training Loss: 0.003495 \tAverage Validation Loss: 0.075978\n",
      "############# Epoch 18  Done   #############\n",
      "\n",
      "############# Epoch 19: Training Start   #############\n",
      "############# Epoch 19: Training End     #############\n",
      "############# Epoch 19: Validation Start   #############\n",
      "############# Epoch 19: Validation End     #############\n",
      "Epoch: 19 \tAvgerage Training Loss: 0.002836 \tAverage Validation Loss: 0.076103\n",
      "############# Epoch 19  Done   #############\n",
      "\n",
      "############# Epoch 20: Training Start   #############\n",
      "############# Epoch 20: Training End     #############\n",
      "############# Epoch 20: Validation Start   #############\n",
      "############# Epoch 20: Validation End     #############\n",
      "Epoch: 20 \tAvgerage Training Loss: 0.002302 \tAverage Validation Loss: 0.082060\n",
      "############# Epoch 20  Done   #############\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# checkpoint_path = '/home/p/parush/bert_data/current_checkpoint_abortion.pt'\n",
    "# best_model = '/home/p/parush/bert_data/best_model_trained_on_abortion.pt'\n",
    "# trained_model,val_targets, val_outputs = train_model(1, EPOCHS, np.Inf, training_loader, validation_loader, model, \n",
    "#                       optimizer,checkpoint_path,best_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d6d1dd18",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "89a4e22e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.5667    0.6071    0.5862        28\n",
      "           1     0.7468    0.8194    0.7815        72\n",
      "\n",
      "   micro avg     0.6972    0.7600    0.7273       100\n",
      "   macro avg     0.6568    0.7133    0.6838       100\n",
      "weighted avg     0.6964    0.7600    0.7268       100\n",
      " samples avg     0.5725    0.5802    0.5751       100\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# val_preds = (np.array(val_outputs) > 0.5).astype(int)\n",
    "\n",
    "# print(classification_report(val_targets, val_preds,labels=[0,1],digits = 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43281ffd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff9e2147",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a02122ed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "783c0a49",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ac80ce4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b80290b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8b08efe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43305844",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1477b03",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c78c285",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44db7497",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f10ef5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "765e4208",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ace94e8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b4ba7a8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "139bb283",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ef4fb45",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f63d12a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f28ed6a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b26195f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e44f3df",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cf5cbdf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40197fd6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11a2d7a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74266b83",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f856c1d9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93224e89",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c259bc10",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81a03ace",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f220bd05",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cdb22a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b27ef23",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41e3b589",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee8bed59",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bd5cb30",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3323d73",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "407914f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1577a3be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a6b1724",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56c3f021",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6150724b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78167564",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb61afe7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "791ce674",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3164884d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7fba559",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlpKernel",
   "language": "python",
   "name": "nlpkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}