Metaphor_Detection_(Tokenlevel).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "mount_file_id": "1HBW3MvJrdK50MJxR3ZKbEgHAoiQdUp27",
      "authorship_tag": "ABX9TyMbmi2oAtwX+8RGMSlXp+yh",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "329fd8dd21b9494d9b8aacbd0cdccc9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_120fa8b9822f4deb850b2a0782f606ea",
              "IPY_MODEL_46e495b9227f4231ae7a9904a5eb56ab",
              "IPY_MODEL_c3778f68ba1946bbb863d81ca1b5f364"
            ],
            "layout": "IPY_MODEL_53f9f3d4b6ce463ab6bb1fcc29a0beee"
          }
        },
        "120fa8b9822f4deb850b2a0782f606ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c64dee80d954a25b9b0ae6edf6ccd36",
            "placeholder": "​",
            "style": "IPY_MODEL_f4b13428f9f84aaca22f0a3ea1c3e360",
            "value": "model.safetensors: 100%"
          }
        },
        "46e495b9227f4231ae7a9904a5eb56ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9599a6df3a5245bd94392a9517a6da1e",
            "max": 1115567652,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_600388f69599490b8c431fa3debb469e",
            "value": 1115567652
          }
        },
        "c3778f68ba1946bbb863d81ca1b5f364": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0343acd622bd404491af724d295357b2",
            "placeholder": "​",
            "style": "IPY_MODEL_d88e26a4ec4a4c3791855ec906a05788",
            "value": " 1.12G/1.12G [00:07&lt;00:00, 110MB/s]"
          }
        },
        "53f9f3d4b6ce463ab6bb1fcc29a0beee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c64dee80d954a25b9b0ae6edf6ccd36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4b13428f9f84aaca22f0a3ea1c3e360": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9599a6df3a5245bd94392a9517a6da1e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "600388f69599490b8c431fa3debb469e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0343acd622bd404491af724d295357b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d88e26a4ec4a4c3791855ec906a05788": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lwachowiak/Multilingual-Metaphor-Detection/blob/main/Metaphor_Detection_(Tokenlevel).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0yMTmZptEkHC"
      },
      "source": [
        "# Imports\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pynvml\n",
        "!pip install transformers[torch]"
      ],
      "metadata": {
        "id": "JamBNJ_frr6T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m9fYtB3_FHuK"
      },
      "source": [
        "#torch and tranformers for model and training\n",
        "import torch\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "from torch.utils.data import TensorDataset\n",
        "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
        "from transformers import AdamW\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "from transformers import DataCollatorForTokenClassification\n",
        "from transformers import Trainer, TrainingArguments\n",
        "import sentencepiece\n",
        "\n",
        "#sklearn for evaluation\n",
        "from sklearn import preprocessing\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "\n",
        "#utilities\n",
        "import pandas as pd\n",
        "import glob, os\n",
        "import time\n",
        "import datetime\n",
        "import random\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import seaborn as sns\n",
        "import pickle         # for saving data structures\n",
        "from pynvml import *  # for checking gpu memory\n",
        "import re"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YyekDMZ28gBc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a730cc4e-ab6a-48dc-cdf5-4246ced98b58"
      },
      "source": [
        "# connect to GPU\n",
        "device = torch.device('cuda')\n",
        "\n",
        "print('Connected to GPU:', torch.cuda.get_device_name(0))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Connected to GPU: Tesla T4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3RPZ14sYHHUm"
      },
      "source": [
        "# Prepare Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sU7NMPaDvbWt"
      },
      "source": [
        "**Functions for preprocessing and creating of Training Data**\n",
        "\n",
        "Originally I used *xlm-roberta-base* as model. Now, there are slightly stronger models available in the same parameter range, for example *microsoft/mdeberta-v3-base*\n",
        "\n",
        "\n",
        "You can try:\n",
        "\n",
        "\n",
        "*   model_name=\"xlm-roberta-base\"\n",
        "*   model_name=\"xlm-roberta-large\"\n",
        "*   model_name=\"microsoft/mdeberta-v3-base\"\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_name=\"xlm-roberta-base\"\n",
        "random_validation=True\n",
        "random_state=1\n",
        "val_percentage=0.1"
      ],
      "metadata": {
        "id": "BVaK3boLMOWT"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Upload files to Google Drive or link to your computer's folder if running locally."
      ],
      "metadata": {
        "id": "zoHbshOctCeU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load datasets\n",
        "test_data=pd.read_csv(\"VUA_test_all.csv\", engine=\"python\")\n",
        "train_data=pd.read_csv(\"VUA_train.csv\", engine=\"python\")"
      ],
      "metadata": {
        "id": "nSQa-1aBnk_g"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def format_for_TokenClf(df) -> list:\n",
        "  data_list = []\n",
        "  sentence=[]\n",
        "  labels=[]\n",
        "  for index, row in df.iterrows():\n",
        "    if row[\"id\"][-2:]==\"_1\" and index!=0:\n",
        "      data_list.append((sentence, labels))\n",
        "      sentence=[]\n",
        "      labels=[]\n",
        "    if row[\"label\"]==1:\n",
        "      label=\"m\"\n",
        "    else:\n",
        "      label=\"l\"\n",
        "    sentence.append(str(row[\"word\"]))\n",
        "    labels.append(label)\n",
        "    if index==len(df)-1:\n",
        "      data_list.append((sentence, labels))\n",
        "\n",
        "  return data_list"
      ],
      "metadata": {
        "id": "mplkMdtDnlHT"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_data=format_for_TokenClf(test_data)\n",
        "train_data=format_for_TokenClf(train_data)\n",
        "print(test_data[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wuqOpet9nlPe",
        "outputId": "601eabdf-a650-4373-b890-d5c9c27dd849"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(['The', 'Labour', 'Party', 'Conference', ':', 'Policy', 'review', 'throws', 'a', 'spanner', 'in', 'the', 'Whitehall', 'machinery'], ['l', 'l', 'l', 'l', 'l', 'l', 'l', 'm', 'l', 'm', 'm', 'l', 'l', 'm'])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_data, val_data= train_test_split(train_data, shuffle=random_validation, test_size=val_percentage, random_state=random_state)"
      ],
      "metadata": {
        "id": "3k9l6nbFcpUB"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Sentences Train: \", len(train_data))\n",
        "print(\"Sentences Val: \", len(val_data))\n",
        "print(\"Sentences Test: \", len(test_data))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i4tXvcFCda7t",
        "outputId": "a8ca571e-576f-49b3-fcb4-54ad7a979be1"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentences Train:  10898\n",
            "Sentences Val:  1211\n",
            "Sentences Test:  4080\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#train\n",
        "train_tags=[tup[1] for tup in train_data]\n",
        "train_texts=[tup[0] for tup in train_data]\n",
        "\n",
        "#val\n",
        "val_tags=[tup[1] for tup in val_data]\n",
        "val_texts=[tup[0] for tup in val_data]\n",
        "\n",
        "#test\n",
        "test_tags=[tup[1] for tup in test_data]\n",
        "test_texts=[tup[0] for tup in test_data]"
      ],
      "metadata": {
        "id": "lKjeYB3ntwBn"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for text in train_tags:\n",
        "  if not isinstance(text, list):\n",
        "    print(text)"
      ],
      "metadata": {
        "id": "S6cd6VWqxNQ9"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(test_texts[0])\n",
        "print(test_tags[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hH9lPm3ivXa5",
        "outputId": "23f9ffa4-3b5c-4f8a-8ffc-88d2c7f20061"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['The', 'Labour', 'Party', 'Conference', ':', 'Policy', 'review', 'throws', 'a', 'spanner', 'in', 'the', 'Whitehall', 'machinery']\n",
            "['l', 'l', 'l', 'l', 'l', 'l', 'l', 'm', 'l', 'm', 'm', 'l', 'l', 'm']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wVxAsANXfpDv"
      },
      "source": [
        "# Tokenize"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ieMHql0gobX"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XYftDnmguJMr"
      },
      "source": [
        "label_list=[\"l\", \"m\"]\n",
        "label_to_id = {l: i for i, l in enumerate(label_list)}\n",
        "num_labels=len(label_list)\n",
        "\n",
        "def tokenize_and_align_labels(texts, tags):\n",
        "  tokenized_inputs = tokenizer(\n",
        "      texts,\n",
        "      padding=True,\n",
        "      truncation=True,\n",
        "      # We use this argument because the texts in our dataset are lists of words (with a label for each word).\n",
        "      is_split_into_words=True,\n",
        "  )\n",
        "  labels = []\n",
        "  for i, label in enumerate(tags):\n",
        "      word_ids = tokenized_inputs.word_ids(batch_index=i)\n",
        "      previous_word_idx = None\n",
        "      label_ids = []\n",
        "      for word_idx in word_ids:\n",
        "          # Special tokens have a word id that is None. We set the label to -100 so they are automatically\n",
        "          # ignored in the loss function.\n",
        "          if word_idx is None:\n",
        "              label_ids.append(-100)\n",
        "          # We set the label for the first token of each word.\n",
        "          elif word_idx != previous_word_idx:\n",
        "              label_ids.append(label_to_id[label[word_idx]])\n",
        "          # For the other tokens in a word, we set the label to either the current label or -100, depending on\n",
        "          # the label_all_tokens flag.\n",
        "          else:\n",
        "              label_ids.append(-100)\n",
        "          previous_word_idx = word_idx\n",
        "\n",
        "      labels.append(label_ids)\n",
        "  tokenized_inputs[\"labels\"] = labels\n",
        "  return tokenized_inputs\n",
        "\n",
        "test_input_and_labels = tokenize_and_align_labels(test_texts, test_tags)\n",
        "\n",
        "val_input_and_labels = tokenize_and_align_labels(val_texts, val_tags)\n",
        "\n",
        "train_input_and_labels = tokenize_and_align_labels(train_texts, train_tags)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6lcPXbZ22yWG"
      },
      "source": [
        "# create dataset\n",
        "class OurDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "test_dataset = OurDataset(test_input_and_labels, test_input_and_labels[\"labels\"])\n",
        "\n",
        "train_dataset = OurDataset(train_input_and_labels, train_input_and_labels[\"labels\"])\n",
        "\n",
        "val_dataset = OurDataset(val_input_and_labels, val_input_and_labels[\"labels\"])\n"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "50I3MZhnod-Y",
        "outputId": "7956cea0-d7ee-4114-e257-d6be57c97839"
      },
      "source": [
        "test_dataset.__getitem__(0)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': tensor([     0,    581,    239,  38648,  31016, 114732,    152,  80042,   8347,\n",
              "         104250,      7,     10,  27734,   1679,     23,     70,  22392,  29907,\n",
              "          36279,   1294,      2,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
              "              1,      1,      1,      1,      1,      1,      1,      1,      1]),\n",
              " 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
              " 'labels': tensor([-100,    0,    0, -100,    0,    0,    0,    0,    0,    1, -100,    0,\n",
              "            1, -100,    1,    0,    0, -100,    1, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100])}"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9miQ8_HxKqGB"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lE8JxaF5T9Yd"
      },
      "source": [
        "# how the validation and test scores are computed\n",
        "\n",
        "def compute_metrics(eval_preds) -> dict:\n",
        "    logits, labels = eval_preds\n",
        "    predictions = np.argmax(logits, axis=-1)\n",
        "\n",
        "    # Remove ignored index (special tokens) and convert to labels\n",
        "    true_labels = [[label_list[l] for l in label if l != -100] for label in labels]\n",
        "    true_predictions = [\n",
        "        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
        "        for prediction, label in zip(predictions, labels)\n",
        "    ]\n",
        "\n",
        "    true_labels = [val for sublist in true_labels for val in sublist]\n",
        "    true_predictions = [val for sublist in true_predictions for val in sublist]\n",
        "\n",
        "    print(classification_report(true_labels, true_predictions))#, target_names=target_names))\n",
        "\n",
        "    precision, recall, f1, _ = precision_recall_fscore_support(true_labels, true_predictions, average=\"weighted\")\n",
        "\n",
        "    return {\n",
        "        \"precision\": precision,\n",
        "        \"recall\": recall,\n",
        "        \"f1\": f1\n",
        "    }"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AcDLjK-i4-Y8",
        "outputId": "8a87ee0f-eea2-47ea-8126-8466c7580fbb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# training arguments\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='./MetaphorExtraction/results',          # output directory\n",
        "    num_train_epochs=8,                                 # total # of training epochs\n",
        "    per_device_train_batch_size=8,                      # batch size per device during training\n",
        "    per_device_eval_batch_size=16,                      # batch size for evaluation\n",
        "    warmup_steps=0,                                     # number of warmup steps for learning rate scheduler\n",
        "    weight_decay=0,                                     # strength of weight decay\n",
        "    learning_rate=2e-5,\n",
        "    logging_dir='./MetaphorExtraction/logs',            # directory for storing logs\n",
        "    evaluation_strategy= \"epoch\",                       # steps or epochs\n",
        "    save_strategy = \"epoch\",\n",
        "    # eval_steps=500,\n",
        "    # save_total_limit=0,\n",
        "    load_best_model_at_end=True,                        #loads the model with the best evaluation score\n",
        "    metric_for_best_model=\"f1\",\n",
        "    greater_is_better=True\n",
        ")"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104,
          "referenced_widgets": [
            "329fd8dd21b9494d9b8aacbd0cdccc9d",
            "120fa8b9822f4deb850b2a0782f606ea",
            "46e495b9227f4231ae7a9904a5eb56ab",
            "c3778f68ba1946bbb863d81ca1b5f364",
            "53f9f3d4b6ce463ab6bb1fcc29a0beee",
            "6c64dee80d954a25b9b0ae6edf6ccd36",
            "f4b13428f9f84aaca22f0a3ea1c3e360",
            "9599a6df3a5245bd94392a9517a6da1e",
            "600388f69599490b8c431fa3debb469e",
            "0343acd622bd404491af724d295357b2",
            "d88e26a4ec4a4c3791855ec906a05788"
          ]
        },
        "id": "yJf_Rnyf26el",
        "outputId": "f4ef5915-ea4a-4e79-9aa1-bc21f4c62d9c"
      },
      "source": [
        "# initialize model\n",
        "model = AutoModelForTokenClassification.from_pretrained(model_name, num_labels=num_labels)\n"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/1.12G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "329fd8dd21b9494d9b8aacbd0cdccc9d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7LPAH8QvBezG",
        "outputId": "67f1ad32-77d9-4d47-e959-268832b4be02"
      },
      "source": [
        "nvmlInit()\n",
        "h = nvmlDeviceGetHandleByIndex(0)\n",
        "info = nvmlDeviceGetMemoryInfo(h)\n",
        "print(f'total    : {info.total}')\n",
        "print(f'free     : {info.free}')\n",
        "print(f'used     : {info.used}')"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total    : 16106127360\n",
            "free     : 15832514560\n",
            "used     : 273612800\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wEg8krxu-MY4"
      },
      "source": [
        "# initialize huggingface trainer\n",
        "trainer = Trainer(\n",
        "        model=model,\n",
        "        args=training_args,\n",
        "        train_dataset = train_dataset,\n",
        "        eval_dataset=val_dataset,\n",
        "        tokenizer=tokenizer,\n",
        "        compute_metrics=compute_metrics,\n",
        "    )"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nZIimqAK28qS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "ccbdd4df-3925-4879-c85f-5b9b28fe516b"
      },
      "source": [
        "# train\n",
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use thePyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 10898\n",
            "  Num Epochs = 8\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 10904\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='10904' max='10904' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [10904/10904 1:57:48, Epoch 8/8]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.158000</td>\n",
              "      <td>0.138226</td>\n",
              "      <td>0.938776</td>\n",
              "      <td>0.942913</td>\n",
              "      <td>0.938748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.119500</td>\n",
              "      <td>0.130923</td>\n",
              "      <td>0.948559</td>\n",
              "      <td>0.950609</td>\n",
              "      <td>0.949253</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.092600</td>\n",
              "      <td>0.134616</td>\n",
              "      <td>0.950395</td>\n",
              "      <td>0.952875</td>\n",
              "      <td>0.950750</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.066900</td>\n",
              "      <td>0.148513</td>\n",
              "      <td>0.951833</td>\n",
              "      <td>0.952875</td>\n",
              "      <td>0.952282</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.051300</td>\n",
              "      <td>0.165712</td>\n",
              "      <td>0.952289</td>\n",
              "      <td>0.953930</td>\n",
              "      <td>0.952871</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.039300</td>\n",
              "      <td>0.203211</td>\n",
              "      <td>0.952352</td>\n",
              "      <td>0.953772</td>\n",
              "      <td>0.952901</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.028000</td>\n",
              "      <td>0.244468</td>\n",
              "      <td>0.950951</td>\n",
              "      <td>0.952981</td>\n",
              "      <td>0.951561</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.020600</td>\n",
              "      <td>0.253753</td>\n",
              "      <td>0.952496</td>\n",
              "      <td>0.954193</td>\n",
              "      <td>0.953077</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1211\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           l       0.95      0.98      0.97     17013\n",
            "           m       0.81      0.58      0.68      1958\n",
            "\n",
            "    accuracy                           0.94     18971\n",
            "   macro avg       0.88      0.78      0.82     18971\n",
            "weighted avg       0.94      0.94      0.94     18971\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./MetaphorExtraction/results/checkpoint-1363\n",
            "Configuration saved in ./MetaphorExtraction/results/checkpoint-1363/config.json\n",
            "Model weights saved in ./MetaphorExtraction/results/checkpoint-1363/pytorch_model.bin\n",
            "tokenizer config file saved in ./MetaphorExtraction/results/checkpoint-1363/tokenizer_config.json\n",
            "Special tokens file saved in ./MetaphorExtraction/results/checkpoint-1363/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1211\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           l       0.97      0.98      0.97     17013\n",
            "           m       0.79      0.70      0.75      1958\n",
            "\n",
            "    accuracy                           0.95     18971\n",
            "   macro avg       0.88      0.84      0.86     18971\n",
            "weighted avg       0.95      0.95      0.95     18971\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./MetaphorExtraction/results/checkpoint-2726\n",
            "Configuration saved in ./MetaphorExtraction/results/checkpoint-2726/config.json\n",
            "Model weights saved in ./MetaphorExtraction/results/checkpoint-2726/pytorch_model.bin\n",
            "tokenizer config file saved in ./MetaphorExtraction/results/checkpoint-2726/tokenizer_config.json\n",
            "Special tokens file saved in ./MetaphorExtraction/results/checkpoint-2726/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1211\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           l       0.96      0.98      0.97     17013\n",
            "           m       0.83      0.68      0.75      1958\n",
            "\n",
            "    accuracy                           0.95     18971\n",
            "   macro avg       0.90      0.83      0.86     18971\n",
            "weighted avg       0.95      0.95      0.95     18971\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./MetaphorExtraction/results/checkpoint-4089\n",
            "Configuration saved in ./MetaphorExtraction/results/checkpoint-4089/config.json\n",
            "Model weights saved in ./MetaphorExtraction/results/checkpoint-4089/pytorch_model.bin\n",
            "tokenizer config file saved in ./MetaphorExtraction/results/checkpoint-4089/tokenizer_config.json\n",
            "Special tokens file saved in ./MetaphorExtraction/results/checkpoint-4089/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1211\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           l       0.97      0.98      0.97     17013\n",
            "           m       0.79      0.74      0.77      1958\n",
            "\n",
            "    accuracy                           0.95     18971\n",
            "   macro avg       0.88      0.86      0.87     18971\n",
            "weighted avg       0.95      0.95      0.95     18971\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./MetaphorExtraction/results/checkpoint-5452\n",
            "Configuration saved in ./MetaphorExtraction/results/checkpoint-5452/config.json\n",
            "Model weights saved in ./MetaphorExtraction/results/checkpoint-5452/pytorch_model.bin\n",
            "tokenizer config file saved in ./MetaphorExtraction/results/checkpoint-5452/tokenizer_config.json\n",
            "Special tokens file saved in ./MetaphorExtraction/results/checkpoint-5452/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1211\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           l       0.97      0.98      0.97     17013\n",
            "           m       0.81      0.73      0.77      1958\n",
            "\n",
            "    accuracy                           0.95     18971\n",
            "   macro avg       0.89      0.85      0.87     18971\n",
            "weighted avg       0.95      0.95      0.95     18971\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./MetaphorExtraction/results/checkpoint-6815\n",
            "Configuration saved in ./MetaphorExtraction/results/checkpoint-6815/config.json\n",
            "Model weights saved in ./MetaphorExtraction/results/checkpoint-6815/pytorch_model.bin\n",
            "tokenizer config file saved in ./MetaphorExtraction/results/checkpoint-6815/tokenizer_config.json\n",
            "Special tokens file saved in ./MetaphorExtraction/results/checkpoint-6815/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1211\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           l       0.97      0.98      0.97     17013\n",
            "           m       0.80      0.74      0.77      1958\n",
            "\n",
            "    accuracy                           0.95     18971\n",
            "   macro avg       0.89      0.86      0.87     18971\n",
            "weighted avg       0.95      0.95      0.95     18971\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./MetaphorExtraction/results/checkpoint-8178\n",
            "Configuration saved in ./MetaphorExtraction/results/checkpoint-8178/config.json\n",
            "Model weights saved in ./MetaphorExtraction/results/checkpoint-8178/pytorch_model.bin\n",
            "tokenizer config file saved in ./MetaphorExtraction/results/checkpoint-8178/tokenizer_config.json\n",
            "Special tokens file saved in ./MetaphorExtraction/results/checkpoint-8178/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1211\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           l       0.97      0.98      0.97     17013\n",
            "           m       0.81      0.71      0.76      1958\n",
            "\n",
            "    accuracy                           0.95     18971\n",
            "   macro avg       0.89      0.84      0.87     18971\n",
            "weighted avg       0.95      0.95      0.95     18971\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./MetaphorExtraction/results/checkpoint-9541\n",
            "Configuration saved in ./MetaphorExtraction/results/checkpoint-9541/config.json\n",
            "Model weights saved in ./MetaphorExtraction/results/checkpoint-9541/pytorch_model.bin\n",
            "tokenizer config file saved in ./MetaphorExtraction/results/checkpoint-9541/tokenizer_config.json\n",
            "Special tokens file saved in ./MetaphorExtraction/results/checkpoint-9541/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1211\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           l       0.97      0.98      0.97     17013\n",
            "           m       0.81      0.73      0.77      1958\n",
            "\n",
            "    accuracy                           0.95     18971\n",
            "   macro avg       0.89      0.85      0.87     18971\n",
            "weighted avg       0.95      0.95      0.95     18971\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./MetaphorExtraction/results/checkpoint-10904\n",
            "Configuration saved in ./MetaphorExtraction/results/checkpoint-10904/config.json\n",
            "Model weights saved in ./MetaphorExtraction/results/checkpoint-10904/pytorch_model.bin\n",
            "tokenizer config file saved in ./MetaphorExtraction/results/checkpoint-10904/tokenizer_config.json\n",
            "Special tokens file saved in ./MetaphorExtraction/results/checkpoint-10904/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./MetaphorExtraction/results/checkpoint-10904 (score: 0.9530768829001109).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=10904, training_loss=0.0736900699199864, metrics={'train_runtime': 7069.4653, 'train_samples_per_second': 12.332, 'train_steps_per_second': 1.542, 'total_flos': 7208018013988224.0, 'train_loss': 0.0736900699199864, 'epoch': 8.0})"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Score on the test set\n",
        "trainer.evaluate(test_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 384
        },
        "id": "xRZMADdvhNrw",
        "outputId": "6fbcea4d-97f6-43ab-a74f-56d29a9411eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 4080\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='255' max='255' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [255/255 01:23]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           l       0.96      0.98      0.97     51540\n",
            "           m       0.82      0.71      0.76      6819\n",
            "\n",
            "    accuracy                           0.95     58359\n",
            "   macro avg       0.89      0.84      0.87     58359\n",
            "weighted avg       0.95      0.95      0.95     58359\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'epoch': 8.0,\n",
              " 'eval_f1': 0.9466133325847397,\n",
              " 'eval_loss': 0.26670947670936584,\n",
              " 'eval_precision': 0.946064396352123,\n",
              " 'eval_recall': 0.9483198821090149,\n",
              " 'eval_runtime': 85.1608,\n",
              " 'eval_samples_per_second': 47.909,\n",
              " 'eval_steps_per_second': 2.994}"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovbaJVNyzw_O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b3e2ca13-2444-47f5-ff89-1309a6f9f8f9"
      },
      "source": [
        "from datetime import date\n",
        "trainer.save_model(\"./saved-models/metaphor_extraction_\"+str(date.today())+\"_randVal-\"+str(random_validation)+\"_\"+model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to ./saved-models/metaphor_extraction_2022-02-24_randVal-True_xlm-roberta-base\n",
            "Configuration saved in ./saved-models/metaphor_extraction_2022-02-24_randVal-True_xlm-roberta-base/config.json\n",
            "Model weights saved in ./saved-models/metaphor_extraction_2022-02-24_randVal-True_xlm-roberta-base/pytorch_model.bin\n",
            "tokenizer config file saved in ./saved-models/metaphor_extraction_2022-02-24_randVal-True_xlm-roberta-base/tokenizer_config.json\n",
            "Special tokens file saved in ./saved-models/metaphor_extraction_2022-02-24_randVal-True_xlm-roberta-base/special_tokens_map.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P_dYhX6t2FsM"
      },
      "source": [
        "# Using the Model for Inference"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline"
      ],
      "metadata": {
        "id": "qnb4ruMEtso9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "label_list= ['literal',\"metaphoric\"]\n",
        "label_dict_relations={ i : l for i, l in enumerate(label_list) }"
      ],
      "metadata": {
        "id": "9fmqFqEDzEid"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "PATH = \"./saved-models/my_model\"\n",
        "model_metaphor_detection = AutoModelForTokenClassification.from_pretrained(PATH, id2label=label_dict_relations)\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ],
      "metadata": {
        "id": "fjvnF0nIumQg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pipeline_metaphors=pipeline(\"ner\", model=model_metaphor_detection, tokenizer=tokenizer, aggregation_strategy=\"simple\")"
      ],
      "metadata": {
        "id": "7kEZyuAbudIG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pipeline_metaphors(\"Our love is at crossroads and the company is going into hibernation. The bear is sleeping well.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FaHMbInvvDex",
        "outputId": "936859c6-98fa-44b9-c106-ea6a52c22868"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'end': 11,\n",
              "  'entity_group': 'l',\n",
              "  'score': 0.99962264,\n",
              "  'start': 0,\n",
              "  'word': 'Our love is'},\n",
              " {'end': 25,\n",
              "  'entity_group': 'metaphoric',\n",
              "  'score': 0.9950348,\n",
              "  'start': 12,\n",
              "  'word': 'at crossroads'},\n",
              " {'end': 44,\n",
              "  'entity_group': 'l',\n",
              "  'score': 0.99978614,\n",
              "  'start': 26,\n",
              "  'word': 'and the company is'},\n",
              " {'end': 67,\n",
              "  'entity_group': 'metaphoric',\n",
              "  'score': 0.86717546,\n",
              "  'start': 45,\n",
              "  'word': 'going into hibernation'},\n",
              " {'end': 95,\n",
              "  'entity_group': 'l',\n",
              "  'score': 0.9984967,\n",
              "  'start': 67,\n",
              "  'word': '. The bear is sleeping well.'}]"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    }
  ]
}