training_spam.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e9ebf294",
   "metadata": {},
   "source": [
    "## Part I: Reading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1a047f17",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pandas as pd\n",
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments, EarlyStoppingCallback\n",
    "from sklearn.model_selection import train_test_split\n",
    "from datasets import Dataset, load_metric\n",
    "import gc\n",
    "from sklearn.metrics import precision_recall_fscore_support,accuracy_score\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_df=pd.read_csv(\"train.csv.zip\", index_col=False)\n",
    "train_df=train_df.drop_duplicates(subset=[\"Tweet\"]).reset_index(drop=True)\n",
    "train_df=train_df.dropna(subset=[\"Tweet\",\"Type\"])\n",
    "# train_df[\"label\"] = train_df[\"Type\"].apply(lambda x: 1 if x == \"Spam\" else 0)\n",
    "train_df[\"label\"] = train_df[\"Type\"]\n",
    "train_df[\"label\"] =train_df[\"label\"].astype(str)\n",
    "train_df[\"Tweet\"] =train_df[\"Tweet\"].astype(str)\n",
    "df=train_df[['Tweet',\"label\",\"Type\"]]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06a55eef",
   "metadata": {},
   "source": [
    "## Part II: Train-test-val split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bf7983ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "def stratified_train_test_val_split(data, label_col=\"label\", train_size=0.8, val_size=0.1, test_size=0.1, random_state=42):\n",
    "    # First, split into 80% training and 20% for testing + validation\n",
    "    train_data, temp_data = train_test_split(\n",
    "        data, test_size=(1 - train_size), stratify=data[label_col], random_state=random_state\n",
    "    )\n",
    "    \n",
    "    # Then, split the 20% (temp_data) into 10% validation and 10% testing\n",
    "    relative_val_size = val_size / (val_size + test_size)  # Adjust size for the remaining 20%\n",
    "    val_data, test_data = train_test_split(\n",
    "        temp_data, test_size=(1 - relative_val_size), stratify=temp_data[label_col], random_state=random_state\n",
    "    )\n",
    "    \n",
    "    return train_data, val_data, test_data\n",
    "\n",
    "# Example usage:\n",
    "train_data, val_data, test_data = stratified_train_test_val_split(df, label_col=\"label\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53ee0273",
   "metadata": {},
   "source": [
    "# Part III: fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4bd5951a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def finetune_tweet_classifier(train_data,val_data, model_name, num_labels=2, epochs=5, batch_size=8):\n",
    "    # Map the \"Type\" column to 0 and 1, ensuring labels are integers\n",
    "    label2id = {\"Spam\": 1, \"Quality\": 0}\n",
    "    train_data[\"label\"] = train_data[\"Type\"].map(label2id).astype(int)\n",
    "    val_data[\"label\"]= val_data[\"Type\"].map(label2id).astype(int)\n",
    "\n",
    "    \n",
    "    # Convert to Hugging Face Dataset\n",
    "    train_dataset = Dataset.from_pandas(train_data)\n",
    "    val_dataset = Dataset.from_pandas(val_data)\n",
    "    \n",
    "    # Tokenizer and model\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "    model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2).to(\"cuda\")\n",
    "\n",
    "    # Tokenize function with padding and truncation\n",
    "    def tokenize_function(example):\n",
    "        return tokenizer(\n",
    "            example[\"Tweet\"],\n",
    "            padding=\"max_length\",  # Ensures all sequences are the same length\n",
    "            truncation=True,       # Truncates sequences longer than model's max length\n",
    "            max_length=512\n",
    "        )\n",
    "    \n",
    "    train_dataset = train_dataset.map(tokenize_function, batched=True)\n",
    "    val_dataset = val_dataset.map(tokenize_function, batched=True)\n",
    "\n",
    "    # Ensure dataset format\n",
    "    train_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "    val_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "    \n",
    "    # Metric calculation\n",
    "    def compute_metrics(pred):\n",
    "        labels = pred.label_ids\n",
    "        preds = pred.predictions.argmax(-1)\n",
    "        precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average=\"weighted\")\n",
    "        acc = accuracy_score(labels, preds)\n",
    "        return {'accuracy': acc, 'f1': f1, 'precision': precision, 'recall': recall}\n",
    "\n",
    "    # Training arguments\n",
    "    training_args = TrainingArguments(\n",
    "        output_dir=\"./results\",\n",
    "        evaluation_strategy=\"epoch\",\n",
    "        save_strategy=\"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        num_train_epochs=epochs,\n",
    "        load_best_model_at_end=True,\n",
    "        metric_for_best_model=\"f1\",\n",
    "        logging_dir='./logs',\n",
    "        logging_steps=1000\n",
    "    )\n",
    "\n",
    "    # Early stopping callback\n",
    "    early_stopping_callback = EarlyStoppingCallback(early_stopping_patience=2)\n",
    "\n",
    "    # Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=training_args,\n",
    "        train_dataset=train_dataset,\n",
    "        eval_dataset=val_dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics,\n",
    "        callbacks=[early_stopping_callback]\n",
    "    )\n",
    "\n",
    "    # Train model\n",
    "    trainer.train()\n",
    "    base_model_name = model_name.split(\"/\")[-1]\n",
    "    new_model_name = f\"{base_model_name}_Twitter_spam_classification\"\n",
    "    \n",
    "    # Save the model and tokenizer\n",
    "    model.save_pretrained(f\"models/{new_model_name}\")\n",
    "    tokenizer.save_pretrained(f\"models/{new_model_name}\")\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d8f3dbb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\charl\\anaconda3\\envs\\pytorch_env\\lib\\site-packages\\transformers\\tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n",
      "Some weights of XLMRobertaForSequenceClassification were not initialized from the model checkpoint at FacebookAI/xlm-roberta-large and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d337f57da2bd4fe2b1e823afb006a604",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/9429 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08dbc0929b1d46389a4e76a9eb816f9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1179 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\charl\\anaconda3\\envs\\pytorch_env\\lib\\site-packages\\transformers\\training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='5895' max='5895' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [5895/5895 33:54, Epoch 5/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.222600</td>\n",
       "      <td>0.169904</td>\n",
       "      <td>0.960136</td>\n",
       "      <td>0.960102</td>\n",
       "      <td>0.961078</td>\n",
       "      <td>0.960136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.134100</td>\n",
       "      <td>0.160637</td>\n",
       "      <td>0.965225</td>\n",
       "      <td>0.965219</td>\n",
       "      <td>0.965326</td>\n",
       "      <td>0.965225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.119200</td>\n",
       "      <td>0.210690</td>\n",
       "      <td>0.969466</td>\n",
       "      <td>0.969456</td>\n",
       "      <td>0.969720</td>\n",
       "      <td>0.969466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.075400</td>\n",
       "      <td>0.212541</td>\n",
       "      <td>0.964377</td>\n",
       "      <td>0.964378</td>\n",
       "      <td>0.964401</td>\n",
       "      <td>0.964377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.047400</td>\n",
       "      <td>0.195720</td>\n",
       "      <td>0.970314</td>\n",
       "      <td>0.970311</td>\n",
       "      <td>0.970344</td>\n",
       "      <td>0.970314</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "XLMRobertaForSequenceClassification(\n",
       "  (roberta): XLMRobertaModel(\n",
       "    (embeddings): XLMRobertaEmbeddings(\n",
       "      (word_embeddings): Embedding(250002, 1024, padding_idx=1)\n",
       "      (position_embeddings): Embedding(514, 1024, padding_idx=1)\n",
       "      (token_type_embeddings): Embedding(1, 1024)\n",
       "      (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): XLMRobertaEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-23): 24 x XLMRobertaLayer(\n",
       "          (attention): XLMRobertaAttention(\n",
       "            (self): XLMRobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): XLMRobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): XLMRobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): XLMRobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (classifier): XLMRobertaClassificationHead(\n",
       "    (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "    (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finetune_tweet_classifier(train_data, val_data, \"FacebookAI/xlm-roberta-large\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "61d1d027",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\charl\\anaconda3\\envs\\pytorch_env\\lib\\site-packages\\transformers\\tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n",
      "C:\\Users\\charl\\anaconda3\\envs\\pytorch_env\\lib\\site-packages\\transformers\\convert_slow_tokenizer.py:551: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
      "  warnings.warn(\n",
      "Some weights of DebertaV2ForSequenceClassification were not initialized from the model checkpoint at microsoft/deberta-v3-large and are newly initialized: ['classifier.bias', 'classifier.weight', 'pooler.dense.bias', 'pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0af0e016fc744e12abb1f853c05a83f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/9429 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ea772b1a5654dd9ac6b960d0fb857e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1179 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\charl\\anaconda3\\envs\\pytorch_env\\lib\\site-packages\\transformers\\training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='11790' max='11790' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [11790/11790 1:00:16, Epoch 5/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.146600</td>\n",
       "      <td>0.169423</td>\n",
       "      <td>0.960984</td>\n",
       "      <td>0.960925</td>\n",
       "      <td>0.962875</td>\n",
       "      <td>0.960984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.117400</td>\n",
       "      <td>0.183220</td>\n",
       "      <td>0.966073</td>\n",
       "      <td>0.966058</td>\n",
       "      <td>0.966493</td>\n",
       "      <td>0.966073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.066800</td>\n",
       "      <td>0.197709</td>\n",
       "      <td>0.970314</td>\n",
       "      <td>0.970301</td>\n",
       "      <td>0.970693</td>\n",
       "      <td>0.970314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.051700</td>\n",
       "      <td>0.166689</td>\n",
       "      <td>0.972858</td>\n",
       "      <td>0.972846</td>\n",
       "      <td>0.973288</td>\n",
       "      <td>0.972858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.029000</td>\n",
       "      <td>0.190577</td>\n",
       "      <td>0.972010</td>\n",
       "      <td>0.972001</td>\n",
       "      <td>0.972306</td>\n",
       "      <td>0.972010</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DebertaV2ForSequenceClassification(\n",
       "  (deberta): DebertaV2Model(\n",
       "    (embeddings): DebertaV2Embeddings(\n",
       "      (word_embeddings): Embedding(128100, 1024, padding_idx=0)\n",
       "      (LayerNorm): LayerNorm((1024,), eps=1e-07, elementwise_affine=True)\n",
       "      (dropout): StableDropout()\n",
       "    )\n",
       "    (encoder): DebertaV2Encoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-23): 24 x DebertaV2Layer(\n",
       "          (attention): DebertaV2Attention(\n",
       "            (self): DisentangledSelfAttention(\n",
       "              (query_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (pos_dropout): StableDropout()\n",
       "              (dropout): StableDropout()\n",
       "            )\n",
       "            (output): DebertaV2SelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-07, elementwise_affine=True)\n",
       "              (dropout): StableDropout()\n",
       "            )\n",
       "          )\n",
       "          (intermediate): DebertaV2Intermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): DebertaV2Output(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-07, elementwise_affine=True)\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (rel_embeddings): Embedding(512, 1024)\n",
       "      (LayerNorm): LayerNorm((1024,), eps=1e-07, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (pooler): ContextPooler(\n",
       "    (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "    (dropout): StableDropout()\n",
       "  )\n",
       "  (classifier): Linear(in_features=1024, out_features=2, bias=True)\n",
       "  (dropout): StableDropout()\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finetune_tweet_classifier(train_data, val_data, \"microsoft/deberta-v3-large\",num_labels=2, epochs=5, batch_size=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56b09724",
   "metadata": {},
   "source": [
    "## Part IV: Evaluation on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "63283845",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10ec2e1102534779911babfe64418ca1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1179 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9745547073791349\n",
      "Precision: 0.9745737843156286\n",
      "Recall: 0.9745547073791349\n",
      "F1-Score: 0.9745530226387936\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def evaluate_model_on_test_data(test_data, model_path, batch_size=8):\n",
    "    # Load the model and tokenizer\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "    model = AutoModelForSequenceClassification.from_pretrained(model_path).to(\"cuda\")\n",
    "    model.eval()  # Set model to evaluation mode\n",
    "    \n",
    "    # Prepare the test data\n",
    "    test_data[\"Tweet\"] = test_data[\"Tweet\"].astype(str)\n",
    "    label2id = {\"Spam\": 1, \"Quality\": 0}\n",
    "    test_data[\"label\"] = test_data[\"Type\"].map(label2id).astype(int)\n",
    "\n",
    "    # Convert to Hugging Face Dataset and tokenize\n",
    "    test_dataset = Dataset.from_pandas(test_data)\n",
    "    def tokenize_function(example):\n",
    "        return tokenizer(\n",
    "            example[\"Tweet\"],\n",
    "            padding=\"max_length\",\n",
    "            truncation=True,\n",
    "            max_length=512\n",
    "        )\n",
    "    \n",
    "    test_dataset = test_dataset.map(tokenize_function, batched=True)\n",
    "    test_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "    \n",
    "    # DataLoader for test set\n",
    "    test_loader = DataLoader(test_dataset, batch_size=batch_size)\n",
    "    \n",
    "    # Make predictions\n",
    "    predictions = []\n",
    "    with torch.no_grad():\n",
    "        for batch in test_loader:\n",
    "            input_ids = batch['input_ids'].to(\"cuda\")\n",
    "            attention_mask = batch['attention_mask'].to(\"cuda\")\n",
    "            \n",
    "            # Forward pass\n",
    "            outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "            logits = outputs.logits\n",
    "            preds = torch.argmax(logits, dim=-1).cpu().numpy()  # Get predicted labels\n",
    "            predictions.extend(preds)\n",
    "    \n",
    "    # Calculate evaluation metrics\n",
    "    true_labels = test_data[\"label\"].tolist()\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(true_labels, predictions, average=\"weighted\")\n",
    "    accuracy = accuracy_score(true_labels, predictions)\n",
    "    \n",
    "    # Print metrics\n",
    "    print(f\"Accuracy: {accuracy}\")\n",
    "    print(f\"Precision: {precision}\")\n",
    "    print(f\"Recall: {recall}\")\n",
    "    print(f\"F1-Score: {f1}\")\n",
    "\n",
    "    # Return metrics in case you need them\n",
    "    return {\n",
    "        \"accuracy\": accuracy,\n",
    "        \"precision\": precision,\n",
    "        \"recall\": recall,\n",
    "        \"f1\": f1\n",
    "    }\n",
    "\n",
    "model_path = \"models/xlm-roberta-large_Twitter_spam_classification\"\n",
    "metrics = evaluate_model_on_test_data(test_data, model_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3654a1d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d04b8eb4d7b4498822a91a2e42f3cc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1179 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9779474130619169\n",
      "Precision: 0.9780790632953945\n",
      "Recall: 0.9779474130619169\n",
      "F1-Score: 0.9779428095129591\n"
     ]
    }
   ],
   "source": [
    "model_path = \"models/deberta-v3-large_Twitter_spam_classification\"\n",
    "metrics = evaluate_model_on_test_data(test_data, model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "773a225b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (pytorch_env)",
   "language": "python",
   "name": "pytorch_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}