Insight-Sentence-Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentence bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import torch\n",
    "import transformers as ppb\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d=pd.read_csv('./Data/Insight_sentence_total.csv',index_col=0)\n",
    "d=d.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Delete empty rows\n",
    "emp=[]\n",
    "for i in range(len(d)):\n",
    "    text=d['Label'][i]\n",
    "    sentence=d['Sentence'][i]\n",
    "    d['Sentence'][i]=str(d['Sentence'][i])\n",
    "    if text=='Removed':\n",
    "        emp.append(i)\n",
    "    if sentence=='' or sentence=='nan':\n",
    "        emp.append(i)\n",
    "\n",
    "d.drop(index=emp, inplace=True)\n",
    "d=d.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d['Label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training_Batch\n",
    "symbol=sum(d['Label'].value_counts())\n",
    "batch_1 = d[:symbol]\n",
    "batch_1['Label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels=batch_1['Label']\n",
    "train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sample_model_list(if suitable to use )\n",
    "model_list=['distilbert-base-cased','distilbert-base-uncased','allenai/scibert_scivocab_cased','allenai/scibert_scivocab_uncased','gpt2','andreas122001/roberta-academic-detector']\n",
    "model_dict={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scibert_scivocab_uncased model\n",
    "model_class, tokenizer_class, pretrained_weights = (ppb.BertModel, ppb.BertTokenizer, 'allenai/scibert_scivocab_uncased')\n",
    "\n",
    "\n",
    "# Load pretrained model/tokenizer\n",
    "tokenizer = tokenizer_class.from_pretrained(pretrained_weights)\n",
    "model = model_class.from_pretrained(pretrained_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sentence tokenizing\n",
    "tokenized = d['Sentence'][0:len(d)].apply((lambda x: tokenizer.encode(x, add_special_tokens=True)))\n",
    "tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = 0\n",
    "for i in tokenized.values:\n",
    "    if len(i) > max_len:\n",
    "        max_len = len(i)\n",
    "\n",
    "padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])\n",
    "padded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.array(padded).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_mask = np.where(padded != 0, 1, 0)\n",
    "attention_mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# device=torch.device('mps')\n",
    "input_ids = torch.tensor(padded)  \n",
    "attention_mask = torch.tensor(attention_mask)\n",
    "# model.to(device)\n",
    "with torch.no_grad():\n",
    "    last_hidden_states = model(input_ids, attention_mask=attention_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = last_hidden_states[0][:,0,:].numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=d['Label'][0:symbol]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features, test_features, train_labels, test_labels = train_test_split(features[0:symbol], labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_labels.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(train_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Grid Search for Parameters\n",
    "# parameters = {'C': np.linspace(0.0001, 100, 20)}\n",
    "tuned_parameters = [\n",
    "    {'C': [1, 10, 100, 1000], 'kernel': ['linear']},\n",
    "    {'C': [1, 10, 100, 1000], 'kernel': ['rbf'], 'gamma': [0.001, 0.0001]},\n",
    "    {'C': [1, 10, 100, 1000], 'kernel': ['poly'], 'degree': [2, 3, 4], 'gamma': [0.001, 0.0001]},\n",
    "    {'C': [1, 10, 100, 1000], 'kernel': ['sigmoid'], 'gamma': [0.001, 0.0001]}\n",
    "    ]\n",
    "grid_search = GridSearchCV(SVC(), tuned_parameters,cv=5)\n",
    "grid_search.fit(train_features, train_labels)\n",
    "\n",
    "print('best parameters: ', grid_search.best_params_)\n",
    "print('best scrores: ', grid_search.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lr_clf = SVC(C=100,gamma=0.0001,kernel='rbf')\n",
    "lr_clf = SVC(C=grid_search.best_params_['C'], degree=grid_search.best_params_['degree'], gamma=grid_search.best_params_['gamma'], kernel=grid_search.best_params_['kernel'])\n",
    "lr_clf.fit(train_features, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred=lr_clf.predict(test_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_clf.score(test_features, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_pred, test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict remaining data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_train_features=features[:symbol]\n",
    "total_pred_features=features[symbol:len(d)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_clf.fit(total_train_features, labels)\n",
    "y_pred_r=lr_clf.predict(total_pred_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(symbol,len(d)):\n",
    "    d['Label'][i]=y_pred_r[i-symbol]\n",
    "\n",
    "d.to_csv('./Data/Insight_label_pred.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}