46a-BERT-Error-Band-for-CreateDebate-Predictions.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"46a-BERT-Error-Band-for-CreateDebate-Predictions.ipynb","private_outputs":true,"provenance":[],"authorship_tag":"ABX9TyOkd1bTW9R97RPAG0UGrU9P"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","source":["__Objective__: Calculate error bands for estimates infered from Bert on CreateDebate dataset\n","\n","__Runtime__: GPU"],"metadata":{"id":"tXx-OijvRObT"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"jsnB6S0-QGQd"},"outputs":[],"source":["!nvidia-smi"]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/gdrive', force_remount=True)"],"metadata":{"id":"hsQUVO2uQmYF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!pip install transformers \n","!pip install datasets"],"metadata":{"id":"b905oc0VQqDZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import os\n","import pandas as pd\n","import numpy as np\n","from tqdm import tqdm \n","import pickle as pkl\n","import matplotlib.pyplot as plt\n","from matplotlib import colors\n","\n","import torch\n","from torch.utils.data import DataLoader\n","from transformers import AdamW\n","from transformers import BertTokenizer, BertForSequenceClassification, pipeline\n","from datasets import load_metric"],"metadata":{"id":"8TYSBjNJQ-rs"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Loading CMV dataset"],"metadata":{"id":"CyQgnsubRdeT"}},{"cell_type":"code","source":["def read_dataset(dir):\n","    \"\"\"Reading texts and labels from dataset\"\"\"\n","    texts_labels = []\n","    with open(dir, 'r', encoding='utf-8') as f:\n","        lines = f.readlines()[1:]\n","        for line in lines:\n","            line = line.strip().split(\",\")\n","            assert(len(line) == 2)\n","            label = line[0].strip()\n","            text = line[1].strip()\n","            if label == 'AH':\n","                texts_labels.append((text, 1))\n","            else:\n","                texts_labels.append((text, 0))\n","    return texts_labels"],"metadata":{"id":"TH3BHEHfRIkK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["dataset = read_dataset('/content/gdrive/MyDrive/DL/dataset/pytorch/train.csv')\n","dataset.extend(read_dataset('/content/gdrive/MyDrive/DL/dataset/pytorch/test.csv'))"],"metadata":{"id":"RXPSouKMRnjz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Training models"],"metadata":{"id":"wsxYWetObxpa"}},{"cell_type":"code","source":["def split_and_merge(dataset):\n","    ah = []\n","    none = [] \n","    for ctext, clabel in dataset:\n","        if (clabel):\n","            ah.append((ctext, clabel)) \n","        else:\n","            none.append((ctext, clabel)) \n","    i = 0 \n","    j = 0 \n","    new_dataset = []\n","    while (i < len(ah) and j < len(none)):\n","        new_dataset.append(ah[i]) \n","        new_dataset.append(none[j])\n","        i += 1\n","        j += 1\n","    while (i < len(ah)):\n","        new_dataset.append(ah[i]) \n","        i += 1 \n","    while (j < len(none)): \n","        new_dataset.append(none[j]) \n","        j += 1 \n","    return new_dataset"],"metadata":{"id":"c2wlPY0yRqiu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# shuffling the dataset \n","dataset = split_and_merge(dataset)\n","\n","# creating folds \n","n_folds = 10\n","fold_length = len(dataset) // n_folds\n","folds = [dataset[i * fold_length: (i + 1) * fold_length] for i in range(n_folds)]"],"metadata":{"id":"GACi0mq9RxtF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# creating tokenizer to get encodings\n","tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') \n","max_seq_length = 64"],"metadata":{"id":"fhBkMIA1R9qu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["folds_text = [] \n","folds_label = [] \n","for i in range(n_folds):\n","    texts = []\n","    labels = [] \n","    for ftext, flabel in folds[i]:\n","        texts.append(ftext)\n","        labels.append(flabel)\n","    folds_text.append(texts)\n","    folds_label.append(labels) "],"metadata":{"id":"0Bd0ixRkSBaF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# generating encodings\n","folds_encoding = [] \n","for i in range(n_folds):\n","    folds_encoding.append(tokenizer(folds_text[i], truncation=True, max_length=max_seq_length, padding=\"max_length\"))"],"metadata":{"id":"tNI8t7PJSGsF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["class CustomDataset(torch.utils.data.Dataset):\n","    def __init__(self, encodings, labels):\n","        self.encodings = encodings\n","        self.labels = labels\n","\n","    def __getitem__(self, idx):\n","        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n","        item['labels'] = torch.tensor(self.labels[idx])\n","        return item\n","\n","    def __len__(self):\n","        return len(self.labels)"],"metadata":{"id":"Lcc-EnuESOff"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["train_dataset = []\n","test_dataset = [] \n","\n","for i in tqdm(range(n_folds)):\n","    cur_text = [] \n","    cur_label = [] \n","    for j in range(n_folds):\n","        if i == j:\n","            continue \n","        cur_text.extend(folds_text[j]) \n","        cur_label.extend(folds_label[j]) \n","    cur_encoding = tokenizer(cur_text, truncation=True, max_length=max_seq_length, padding=\"max_length\")\n","    train_dataset.append(CustomDataset(cur_encoding, cur_label)) \n","    test_dataset.append(CustomDataset(folds_encoding[i], folds_label[i]))"],"metadata":{"id":"7Aaff-gMSYSo"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')"],"metadata":{"id":"DUWedxUfScc-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def get_model(fold_id):\n","    model = BertForSequenceClassification.from_pretrained('bert-base-uncased')\n","    model.to(device)\n","    model.train()\n","\n","    train_loader = DataLoader(train_dataset[fold_id], batch_size=64, shuffle=True)\n","    optim = AdamW(model.parameters(), lr=5e-5)\n","\n","    for epoch in range(3):\n","        for batch in tqdm(train_loader):\n","            optim.zero_grad()\n","            input_ids = batch['input_ids'].to(device)\n","            attention_mask = batch['attention_mask'].to(device)\n","            labels = batch['labels'].to(device)\n","            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n","            loss = outputs[0]\n","            loss.backward()\n","            optim.step()\n","    \n","    return model"],"metadata":{"id":"JT45QUOCTAW-"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Loading CreateDebate dataset"],"metadata":{"id":"Cxo8TzF3b2Xo"}},{"cell_type":"code","source":["!git clone https://github.com/utkarsh512/CreateDebateScraper.git"],"metadata":{"id":"5G6_DvoJT0Z6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["%cd CreateDebateScraper/src/nested/"],"metadata":{"id":"FnpC-ZTJb8DL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from thread import Thread, Comment # from CreateDebateScraper\n","import pickle\n","from copy import deepcopy"],"metadata":{"id":"y1QA4_Hnb-v6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["comments = dict()\n","\n","# topical forums on CreateDebate\n","categories = ['business', 'comedy', 'entertainment', 'health', 'law', 'nsfw',\n","              'politics2', 'religion', 'science', 'shopping', 'sports',\n","              'technology', 'travel', 'world']\n","\n","# topical forums we're interested in!\n","categories_selected = ['politics2', 'religion', 'world', 'science', 'law', 'technology']\n","categories_labels = ['politics', 'religion', 'world', 'science', 'law', 'technology']\n","\n","for x in categories_selected:\n","    comments[x] = list()"],"metadata":{"id":"0EFjkMWXcBiS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# loading dataset from drive for interesting topical forums\n","for cat in tqdm(categories_selected):\n","    fp = open('/content/gdrive/MyDrive/DL/CreateDebate/' + cat + '/threads.log', 'rb')\n","    threads = list()\n","    try:\n","        while True:\n","            e = pickle.load(fp)\n","            threads.append(e)\n","    except EOFError:\n","        fp.close()\n","    #print(f'{cat} - {len(threads)}')\n","    authors = dict()\n","    for thread in threads:\n","        for k, v in thread.comments.items():\n","            try:\n","                authors[v.author].append(v)\n","            except:\n","                authors[v.author] = list()\n","                authors[v.author].append(v)\n","    ctr = 0\n","    with open('/content/gdrive/MyDrive/DL/CreateDebate/' + cat + '/comments_with_score.log', 'rb') as fp:\n","        cws = pickle.load(fp)\n","    for author in authors.keys():\n","        for i in range(len(authors[author])):\n","            comment = authors[author][i]\n","            foo = deepcopy(comment.__dict__)\n","            foo['tag'] = cat\n","            #foo['score'] = cws[ctr][0]\n","            #foo['validation'] = cws[ctr][1][0]\n","            comments[cat].append(foo)\n","            ctr += 1"],"metadata":{"id":"lVPlolJ0cLJd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["comments['politics2'][0].keys()"],"metadata":{"id":"m3JkZJs2czE5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["texts = list()\n","\n","for cat in categories_selected:\n","    for comment in comments[cat]:\n","        texts.append(comment['body'])"],"metadata":{"id":"PPJE6usWc9TR"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Infering class labels"],"metadata":{"id":"Bo0WT7lLn0SI"}},{"cell_type":"code","source":["class InferDataset(torch.utils.data.Dataset):\n","    def __init__(self, texts):\n","        self.texts = texts\n","\n","    def __getitem__(self, idx):\n","        return self.texts[idx]\n","\n","    def __len__(self):\n","        return len(self.texts)"],"metadata":{"id":"nmTZFdbieMon"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["in_data = InferDataset(texts)"],"metadata":{"id":"UHNJ0ZObfgP6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["model = get_model(fold_id=9)"],"metadata":{"id":"VUJMFX0RO6Ql"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["pipe = pipeline(task='text-classification', model=model, tokenizer=tokenizer, device=0)"],"metadata":{"id":"atg1dpQ4fnm5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def classify(text, **kwargs):\n","    return pipe(text, max_length=64, truncation=True, **kwargs)"],"metadata":{"id":"61k9zC2kgEbP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["lbl = list()\n","for out in tqdm(classify(in_data, batch_size=128), total=len(in_data)):\n","    lbl.append(out)\n","with open('/content/gdrive/MyDrive/DL/CreateDebate/errorband/label9.log', 'wb') as f:\n","    pickle.dump(lbl, f)"],"metadata":{"id":"21jx3fD8z-RL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(lbl)"],"metadata":{"id":"vPqCaYYF6JS1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[""],"metadata":{"id":"7Yq9nR04aEkC"},"execution_count":null,"outputs":[]}]}