lstm-ner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "from tensorflow.keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>lemma</th>\n",
       "      <th>next-lemma</th>\n",
       "      <th>next-next-lemma</th>\n",
       "      <th>next-next-pos</th>\n",
       "      <th>next-next-shape</th>\n",
       "      <th>next-next-word</th>\n",
       "      <th>next-pos</th>\n",
       "      <th>next-shape</th>\n",
       "      <th>next-word</th>\n",
       "      <th>...</th>\n",
       "      <th>prev-prev-lemma</th>\n",
       "      <th>prev-prev-pos</th>\n",
       "      <th>prev-prev-shape</th>\n",
       "      <th>prev-prev-word</th>\n",
       "      <th>prev-shape</th>\n",
       "      <th>prev-word</th>\n",
       "      <th>sentence_idx</th>\n",
       "      <th>shape</th>\n",
       "      <th>word</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>thousand</td>\n",
       "      <td>of</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>...</td>\n",
       "      <td>__start2__</td>\n",
       "      <td>__START2__</td>\n",
       "      <td>wildcard</td>\n",
       "      <td>__START2__</td>\n",
       "      <td>wildcard</td>\n",
       "      <td>__START1__</td>\n",
       "      <td>1</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>of</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>have</td>\n",
       "      <td>VBP</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>...</td>\n",
       "      <td>__start1__</td>\n",
       "      <td>__START1__</td>\n",
       "      <td>wildcard</td>\n",
       "      <td>__START1__</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>1</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>have</td>\n",
       "      <td>march</td>\n",
       "      <td>VBN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>marched</td>\n",
       "      <td>VBP</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>...</td>\n",
       "      <td>thousand</td>\n",
       "      <td>NNS</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>1</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>have</td>\n",
       "      <td>march</td>\n",
       "      <td>through</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>through</td>\n",
       "      <td>VBN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>marched</td>\n",
       "      <td>...</td>\n",
       "      <td>of</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>1</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>march</td>\n",
       "      <td>through</td>\n",
       "      <td>london</td>\n",
       "      <td>NNP</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>London</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>through</td>\n",
       "      <td>...</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>1</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>marched</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0     lemma next-lemma next-next-lemma next-next-pos  \\\n",
       "0           0  thousand         of        demonstr           NNS   \n",
       "1           1        of   demonstr            have           VBP   \n",
       "2           2  demonstr       have           march           VBN   \n",
       "3           3      have      march         through            IN   \n",
       "4           4     march    through          london           NNP   \n",
       "\n",
       "  next-next-shape next-next-word next-pos next-shape      next-word  ...  \\\n",
       "0       lowercase  demonstrators       IN  lowercase             of  ...   \n",
       "1       lowercase           have      NNS  lowercase  demonstrators  ...   \n",
       "2       lowercase        marched      VBP  lowercase           have  ...   \n",
       "3       lowercase        through      VBN  lowercase        marched  ...   \n",
       "4     capitalized         London       IN  lowercase        through  ...   \n",
       "\n",
       "  prev-prev-lemma prev-prev-pos prev-prev-shape prev-prev-word   prev-shape  \\\n",
       "0      __start2__    __START2__        wildcard     __START2__     wildcard   \n",
       "1      __start1__    __START1__        wildcard     __START1__  capitalized   \n",
       "2        thousand           NNS     capitalized      Thousands    lowercase   \n",
       "3              of            IN       lowercase             of    lowercase   \n",
       "4        demonstr           NNS       lowercase  demonstrators    lowercase   \n",
       "\n",
       "       prev-word sentence_idx        shape           word tag  \n",
       "0     __START1__            1  capitalized      Thousands   O  \n",
       "1      Thousands            1    lowercase             of   O  \n",
       "2             of            1    lowercase  demonstrators   O  \n",
       "3  demonstrators            1    lowercase           have   O  \n",
       "4           have            1    lowercase        marched   O  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"ner.csv\",encoding = \"ISO-8859-1\",nrows = 50000)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset=data.drop(['Unnamed: 0', 'lemma', 'next-lemma', 'next-next-lemma', 'next-next-pos',\n",
    "       'next-next-shape', 'next-next-word', 'next-pos', 'next-shape',\n",
    "       'next-word', 'prev-iob', 'prev-lemma', 'prev-pos',\n",
    "       'prev-prev-iob', 'prev-prev-lemma', 'prev-prev-pos', 'prev-prev-shape',\n",
    "       'prev-prev-word', 'prev-shape', 'prev-word',\"pos\",\"shape\"],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = list(set(dataset[\"word\"].values))\n",
    "words.append(\"ENDPAD\")\n",
    "tags = list(set(dataset[\"tag\"].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7465"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_words = len(words); n_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['B-art',\n",
       " 'B-gpe',\n",
       " 'B-per',\n",
       " 'I-org',\n",
       " 'B-tim',\n",
       " 'I-art',\n",
       " 'I-per',\n",
       " 'B-org',\n",
       " 'I-gpe',\n",
       " 'B-nat',\n",
       " 'I-tim',\n",
       " 'B-geo',\n",
       " 'I-nat',\n",
       " 'I-geo',\n",
       " 'I-eve',\n",
       " 'O',\n",
       " 'B-eve']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_tags = len(tags); n_tags\n",
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SentenceGetter(object):\n",
    "    \n",
    "    def __init__(self, dataset):\n",
    "        self.n_sent = 1\n",
    "        self.dataset = dataset\n",
    "        self.empty = False\n",
    "        agg_func = lambda s: [(w, t) for w,t in zip(s[\"word\"].values.tolist(),\n",
    "                                                        s[\"tag\"].values.tolist())]\n",
    "        self.grouped = self.dataset.groupby(\"sentence_idx\").apply(agg_func)\n",
    "        self.sentences = [s for s in self.grouped]\n",
    "    \n",
    "    def get_next(self):\n",
    "        try:\n",
    "            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n",
    "            self.n_sent += 1\n",
    "            return s\n",
    "        except:\n",
    "            return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "getter = SentenceGetter(dataset)\n",
    "sentences = getter.sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = {w: i for i, w in enumerate(words)}\n",
    "tag2idx = {t: i for i, t in enumerate(tags)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "X = [[word2idx[w[0]] for w in s] for s in sentences]\n",
    "X = pad_sequences(maxlen=140, sequences=X, padding=\"post\",value=n_words - 1)\n",
    "y = [[tag2idx[w[1]] for w in s] for s in sentences]\n",
    "y = pad_sequences(maxlen=140, sequences=y, padding=\"post\", value=tag2idx[\"O\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical\n",
    "y = [to_categorical(i, num_classes=n_tags) for i in y-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input\n",
    "from tensorflow.keras.models import Model\n",
    "from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional\n",
    "\n",
    "input = Input(shape=(140,))\n",
    "model = Embedding(input_dim=n_words, output_dim=140, input_length=140)(input)\n",
    "model = Dropout(0.1)(model)\n",
    "model = Bidirectional(LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)\n",
    "out = TimeDistributed(Dense(n_tags, activation=\"softmax\"))(model)  # softmax output layer\n",
    "model = Model(input, out)\n",
    "model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38/38 [==============================] - 6s 98ms/step - loss: 0.6802 - accuracy: 0.9492 - val_loss: 0.1530 - val_accuracy: 0.9749\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, np.array(y_train), batch_size=32, epochs=1, validation_split=0.2, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 1s 20ms/step\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "preds = []\n",
    "p = np.argmax(y_pred, axis=-1)\n",
    "for x in p[0]:\n",
    "    preds.append(tags[x])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tags = []\n",
    "t = np.argmax(y_test, axis=-1)\n",
    "for x in t[0]:\n",
    "    test_tags.append(tags[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       B-art       0.00      0.00      0.00         0\n",
      "       I-eve       1.00      0.98      0.99       140\n",
      "       I-org       0.00      0.00      0.00         0\n",
      "       I-tim       0.00      0.00      0.00         0\n",
      "\n",
      "    accuracy                           0.98       140\n",
      "   macro avg       0.25      0.24      0.25       140\n",
      "weighted avg       1.00      0.98      0.99       140\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/prathiba/pra/VIT Chennai/paper/code/.environment/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/prathiba/pra/VIT Chennai/paper/code/.environment/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/prathiba/pra/VIT Chennai/paper/code/.environment/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(classification_report(preds, test_tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8857142857142857"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracy_score(preds, test_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 5 Complete [00h 00m 08s]\n",
      "val_accuracy: 0.9765507578849792\n",
      "\n",
      "Best val_accuracy So Far: 0.9765507578849792\n",
      "Total elapsed time: 00h 01m 03s\n",
      "INFO:tensorflow:Oracle triggered exit\n",
      "Best Hyperparameters:\n",
      "<keras_tuner.engine.hyperparameters.hyperparameters.HyperParameters object at 0x7f6657181950>\n",
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_2 (InputLayer)        [(None, 140)]             0         \n",
      "                                                                 \n",
      " embedding_1 (Embedding)     (None, 140, 140)          1045100   \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 140, 140)          0         \n",
      "                                                                 \n",
      " bidirectional_1 (Bidirecti  (None, 140, 256)          275456    \n",
      " onal)                                                           \n",
      "                                                                 \n",
      " time_distributed_1 (TimeDi  (None, 140, 17)           4369      \n",
      " stributed)                                                      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1324925 (5.05 MB)\n",
      "Trainable params: 1324925 (5.05 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n",
      "Epoch 1/5\n",
      "38/38 [==============================] - 7s 131ms/step - loss: 0.5912 - accuracy: 0.9500 - val_loss: 0.1374 - val_accuracy: 0.9766\n",
      "Epoch 2/5\n",
      "38/38 [==============================] - 5s 129ms/step - loss: 0.1288 - accuracy: 0.9760 - val_loss: 0.1191 - val_accuracy: 0.9766\n",
      "Epoch 3/5\n",
      "38/38 [==============================] - 5s 126ms/step - loss: 0.1161 - accuracy: 0.9760 - val_loss: 0.1137 - val_accuracy: 0.9766\n",
      "Epoch 4/5\n",
      "38/38 [==============================] - 5s 126ms/step - loss: 0.1090 - accuracy: 0.9760 - val_loss: 0.1069 - val_accuracy: 0.9766\n",
      "Epoch 5/5\n",
      "38/38 [==============================] - 5s 125ms/step - loss: 0.0995 - accuracy: 0.9760 - val_loss: 0.0975 - val_accuracy: 0.9766\n"
     ]
    }
   ],
   "source": [
    "from kerastuner.tuners import RandomSearch\n",
    "from kerastuner.engine.hyperparameters import HyperParameters\n",
    "\n",
    "def build_model(hp):\n",
    "    input = Input(shape=(140,))\n",
    "    model = Embedding(input_dim=n_words, output_dim=140, input_length=140)(input)\n",
    "    model = Dropout(hp.Float('Dropout_rate',min_value=0,max_value=0.5,step=0.1))(model)\n",
    "    model = Bidirectional(LSTM(units=hp.Int('lstm_units', min_value=32, max_value=128, step=32), return_sequences=True, recurrent_dropout=hp.Float('dropout', min_value=0.1, max_value=0.5, step=0.1)))(model)\n",
    "    out = TimeDistributed(Dense(n_tags, activation=hp.Choice('dense_activation',values=['relu','sigmoid','softmax'])))(model) \n",
    "    model = Model(input, out)\n",
    "    model.compile(optimizer=\"adam\", loss=hp.Choice('loss_fn',values=['binary_crossentropy','categorical_crossentropy']), metrics=[\"accuracy\"])\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "# Initialize Keras Tuner RandomSearch\n",
    "tuner = RandomSearch(\n",
    "    build_model,\n",
    "    objective='val_accuracy',\n",
    "    max_trials=5,\n",
    "    directory='tuner_dir',\n",
    "    project_name='lstm_sentiment'\n",
    ")\n",
    "\n",
    "# Perform hyperparameter search\n",
    "tuner.search(X_train, np.array(y_train), validation_split=0.2, epochs=3)\n",
    "\n",
    "# Get the best hyperparameters\n",
    "best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]\n",
    "print(\"Best Hyperparameters:\")\n",
    "print(best_hps)\n",
    "\n",
    "# Build the final model with the best hyperparameters\n",
    "model = tuner.hypermodel.build(best_hps)\n",
    "history = model.fit(X_train, np.array(y_train), batch_size=32, epochs=5, validation_split=0.2, verbose=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chosen number of LSTM units: 128\n",
      "Chosen dropout rate: 0.1\n",
      "Chosen Activation Function: softmax\n",
      "Chosen Loss Function: categorical_crossentropy\n"
     ]
    }
   ],
   "source": [
    "# Print the chosen activation function and loss function\n",
    "best_activation = best_hps.get('dense_activation')\n",
    "best_loss_function = best_hps.get('loss_fn')\n",
    "best_units = best_hps.get('lstm_units')\n",
    "best_dropout = best_hps.get('Dropout_rate')\n",
    "print(\"Chosen number of LSTM units:\", best_units)\n",
    "print(\"Chosen dropout rate:\", best_dropout)\n",
    "print(\"Chosen Activation Function:\", best_activation)\n",
    "print(\"Chosen Loss Function:\", best_loss_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 1s 26ms/step\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "preds = []\n",
    "p = np.argmax(y_pred, axis=-1)\n",
    "for x in p[0]:\n",
    "    preds.append(tags[x])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tags = []\n",
    "t = np.argmax(y_test, axis=-1)\n",
    "for x in t[0]:\n",
    "    test_tags.append(tags[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       B-art       0.00      0.00      0.00         0\n",
      "       B-gpe       0.00      0.00      0.00         0\n",
      "       I-art       0.00      0.00      0.00         0\n",
      "       I-eve       1.00      0.89      0.94       140\n",
      "       I-nat       0.00      0.00      0.00         0\n",
      "       I-org       0.00      0.00      0.00         0\n",
      "       I-tim       0.00      0.00      0.00         0\n",
      "\n",
      "    accuracy                           0.89       140\n",
      "   macro avg       0.14      0.13      0.13       140\n",
      "weighted avg       1.00      0.89      0.94       140\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/prathiba/pra/VIT Chennai/paper/code/.environment/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/prathiba/pra/VIT Chennai/paper/code/.environment/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/prathiba/pra/VIT Chennai/paper/code/.environment/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(classification_report(preds, test_tags))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".environment",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}