AS_BERT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install transformers\n",
    "# !pip install seqeval\n",
    "# !pip install tensorboardx\n",
    "# !pip install simpletransformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from simpletransformers.classification import ClassificationModel\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "\n",
    "def f1_multiclass(labels, preds):\n",
    "    return f1_score(labels, preds, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data and format it for simpletransformers\n",
    "\n",
    "train_df = pd.read_json(\"data/AS/train.json\", orient=\"records\")\n",
    "val_df = pd.read_json(\"data/AS/validation.json\", orient=\"records\")\n",
    "test_df = pd.read_json(\"data/AS/test.json\", orient=\"records\")\n",
    "\n",
    "train_df = train_df.loc[:,[\"problem\",\"label\"]]\n",
    "train_df = train_df.rename(columns = {\"problem\":\"text\", \"label\":\"labels\"})\n",
    "val_df = val_df.loc[:,[\"problem\",\"label\"]]\n",
    "val_df = val_df.rename(columns = {\"problem\":\"text\", \"label\":\"labels\"})\n",
    "test_df = test_df.loc[:,[\"problem\",\"label\"]]\n",
    "test_df = test_df.rename(columns = {\"problem\":\"text\", \"label\":\"labels\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>products have similarities which can be analyz...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>learn filters in order to capture local corr...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a successful point cloud registration often li...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>integrating large intelligent reflecting surfa...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the indicator diagram is an important basis to...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44601</th>\n",
       "      <td>semantic segmentation is an important visual p...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44602</th>\n",
       "      <td>people start posting tweets containing texts, ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44603</th>\n",
       "      <td>for the investigation of the steganography, mo...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44604</th>\n",
       "      <td>semantic text matching is one of the most impo...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44605</th>\n",
       "      <td>the deep  has significantly raised the perform...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44606 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  labels\n",
       "0      products have similarities which can be analyz...       8\n",
       "1        learn filters in order to capture local corr...       5\n",
       "2      a successful point cloud registration often li...       5\n",
       "3      integrating large intelligent reflecting surfa...       5\n",
       "4      the indicator diagram is an important basis to...       2\n",
       "...                                                  ...     ...\n",
       "44601  semantic segmentation is an important visual p...       5\n",
       "44602  people start posting tweets containing texts, ...       3\n",
       "44603  for the investigation of the steganography, mo...       5\n",
       "44604  semantic text matching is one of the most impo...       0\n",
       "44605  the deep  has significantly raised the perform...       5\n",
       "\n",
       "[44606 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.7940542946150422,\n",
       " 1.528911739502999,\n",
       " 1.517985366683682,\n",
       " 1.448481896411755,\n",
       " 1.7090421455938696,\n",
       " 0.20291595587399067,\n",
       " 5.377456298975286,\n",
       " 0.4881374480192602,\n",
       " 0.72530081300813,\n",
       " 1.5336427711878975,\n",
       " 0.9240936399419929,\n",
       " 3.8075970977379425,\n",
       " 8.1921028466483,\n",
       " 3.8027280477408354,\n",
       " 4.366715614292707]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate class weights\n",
    "num_classes = 15\n",
    "weights_dict = {i: ((len(train_df)/num_classes)/train_df[\"labels\"].value_counts()[i]) for i in train_df[\"labels\"].value_counts().index}\n",
    "weights = [0]*num_classes\n",
    "for i in weights_dict:\n",
    "    weights[i] = weights_dict[i]\n",
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model parameters\n",
    "args = {\n",
    "    \"train_batch_size\": 32,\n",
    "    \"num_train_epochs\": 21,\n",
    "    \"learning_rate\": 4e-5,\n",
    "#     \"weight\": weights,\n",
    "    \"save_model_every_epoch\": False,\n",
    "    \"save_eval_checkpoints\": False,\n",
    "    \"overwrite_output_dir\": True,\n",
    "    \"reprocess_input_data\": False,\n",
    "    'evaluate_during_training': True,\n",
    "    \"eval_batch_size\": 32\n",
    "}\n",
    "\n",
    "# Create a ClassificationModel\n",
    "model = ClassificationModel('bert', 'allenai/scibert_scivocab_uncased', weight=weights, num_labels=num_classes, args=args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train model\n",
    "\n",
    "model.train_model(train_df, eval_df=val_df, f1=f1_multiclass, acc=accuracy_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#evaluate model\n",
    "result, model_outputs, wrong_predictions = model.eval_model(test_df, f1=f1_multiclass, acc=accuracy_score)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get predictions\n",
    "preds, raw = model.predict(list(test_df[\"text\"]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}