01_data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b416ea40",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from transformers import AutoTokenizer\n",
    "from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer\n",
    "from transformers import DataCollatorForTokenClassification\n",
    "import random\n",
    "import json\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b549a56c",
   "metadata": {},
   "source": [
    "### Read the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cb682bd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Skipping line 281837: expected 25 fields, saw 34\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>lemma</th>\n",
       "      <th>next-lemma</th>\n",
       "      <th>next-next-lemma</th>\n",
       "      <th>next-next-pos</th>\n",
       "      <th>next-next-shape</th>\n",
       "      <th>next-next-word</th>\n",
       "      <th>next-pos</th>\n",
       "      <th>next-shape</th>\n",
       "      <th>next-word</th>\n",
       "      <th>...</th>\n",
       "      <th>prev-prev-lemma</th>\n",
       "      <th>prev-prev-pos</th>\n",
       "      <th>prev-prev-shape</th>\n",
       "      <th>prev-prev-word</th>\n",
       "      <th>prev-shape</th>\n",
       "      <th>prev-word</th>\n",
       "      <th>sentence_idx</th>\n",
       "      <th>shape</th>\n",
       "      <th>word</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>thousand</td>\n",
       "      <td>of</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>...</td>\n",
       "      <td>__start2__</td>\n",
       "      <td>__START2__</td>\n",
       "      <td>wildcard</td>\n",
       "      <td>__START2__</td>\n",
       "      <td>wildcard</td>\n",
       "      <td>__START1__</td>\n",
       "      <td>1.0</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>of</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>have</td>\n",
       "      <td>VBP</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>...</td>\n",
       "      <td>__start1__</td>\n",
       "      <td>__START1__</td>\n",
       "      <td>wildcard</td>\n",
       "      <td>__START1__</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>have</td>\n",
       "      <td>march</td>\n",
       "      <td>VBN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>marched</td>\n",
       "      <td>VBP</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>...</td>\n",
       "      <td>thousand</td>\n",
       "      <td>NNS</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>have</td>\n",
       "      <td>march</td>\n",
       "      <td>through</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>through</td>\n",
       "      <td>VBN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>marched</td>\n",
       "      <td>...</td>\n",
       "      <td>of</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>march</td>\n",
       "      <td>through</td>\n",
       "      <td>london</td>\n",
       "      <td>NNP</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>London</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>through</td>\n",
       "      <td>...</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>marched</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050790</th>\n",
       "      <td>1048570</td>\n",
       "      <td>they</td>\n",
       "      <td>respond</td>\n",
       "      <td>to</td>\n",
       "      <td>TO</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>to</td>\n",
       "      <td>VBD</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>responded</td>\n",
       "      <td>...</td>\n",
       "      <td>forc</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>forces</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>said</td>\n",
       "      <td>47959.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>they</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050791</th>\n",
       "      <td>1048571</td>\n",
       "      <td>respond</td>\n",
       "      <td>to</td>\n",
       "      <td>the</td>\n",
       "      <td>DT</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>the</td>\n",
       "      <td>TO</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>to</td>\n",
       "      <td>...</td>\n",
       "      <td>said</td>\n",
       "      <td>VBD</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>said</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>they</td>\n",
       "      <td>47959.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>responded</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050792</th>\n",
       "      <td>1048572</td>\n",
       "      <td>to</td>\n",
       "      <td>the</td>\n",
       "      <td>attack</td>\n",
       "      <td>NN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>attack</td>\n",
       "      <td>DT</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>the</td>\n",
       "      <td>...</td>\n",
       "      <td>they</td>\n",
       "      <td>PRP</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>they</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>responded</td>\n",
       "      <td>47959.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>to</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050793</th>\n",
       "      <td>1048573</td>\n",
       "      <td>the</td>\n",
       "      <td>attack</td>\n",
       "      <td>with</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>with</td>\n",
       "      <td>NN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>attack</td>\n",
       "      <td>...</td>\n",
       "      <td>respond</td>\n",
       "      <td>VBD</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>responded</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>to</td>\n",
       "      <td>47959.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>the</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050794</th>\n",
       "      <td>1048574</td>\n",
       "      <td>attack</td>\n",
       "      <td>with</td>\n",
       "      <td>machine-gun</td>\n",
       "      <td>JJ</td>\n",
       "      <td>contains-hyphen</td>\n",
       "      <td>machine-gun</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>with</td>\n",
       "      <td>...</td>\n",
       "      <td>to</td>\n",
       "      <td>TO</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>to</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>the</td>\n",
       "      <td>47959.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>attack</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1050795 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Unnamed: 0     lemma next-lemma next-next-lemma next-next-pos  \\\n",
       "0                 0  thousand         of        demonstr           NNS   \n",
       "1                 1        of   demonstr            have           VBP   \n",
       "2                 2  demonstr       have           march           VBN   \n",
       "3                 3      have      march         through            IN   \n",
       "4                 4     march    through          london           NNP   \n",
       "...             ...       ...        ...             ...           ...   \n",
       "1050790     1048570      they    respond              to            TO   \n",
       "1050791     1048571   respond         to             the            DT   \n",
       "1050792     1048572        to        the          attack            NN   \n",
       "1050793     1048573       the     attack            with            IN   \n",
       "1050794     1048574    attack       with     machine-gun            JJ   \n",
       "\n",
       "         next-next-shape next-next-word next-pos next-shape      next-word  \\\n",
       "0              lowercase  demonstrators       IN  lowercase             of   \n",
       "1              lowercase           have      NNS  lowercase  demonstrators   \n",
       "2              lowercase        marched      VBP  lowercase           have   \n",
       "3              lowercase        through      VBN  lowercase        marched   \n",
       "4            capitalized         London       IN  lowercase        through   \n",
       "...                  ...            ...      ...        ...            ...   \n",
       "1050790        lowercase             to      VBD  lowercase      responded   \n",
       "1050791        lowercase            the       TO  lowercase             to   \n",
       "1050792        lowercase         attack       DT  lowercase            the   \n",
       "1050793        lowercase           with       NN  lowercase         attack   \n",
       "1050794  contains-hyphen    machine-gun       IN  lowercase           with   \n",
       "\n",
       "         ... prev-prev-lemma prev-prev-pos prev-prev-shape prev-prev-word  \\\n",
       "0        ...      __start2__    __START2__        wildcard     __START2__   \n",
       "1        ...      __start1__    __START1__        wildcard     __START1__   \n",
       "2        ...        thousand           NNS     capitalized      Thousands   \n",
       "3        ...              of            IN       lowercase             of   \n",
       "4        ...        demonstr           NNS       lowercase  demonstrators   \n",
       "...      ...             ...           ...             ...            ...   \n",
       "1050790  ...            forc           NNS       lowercase         forces   \n",
       "1050791  ...            said           VBD       lowercase           said   \n",
       "1050792  ...            they           PRP       lowercase           they   \n",
       "1050793  ...         respond           VBD       lowercase      responded   \n",
       "1050794  ...              to            TO       lowercase             to   \n",
       "\n",
       "          prev-shape      prev-word sentence_idx        shape           word  \\\n",
       "0           wildcard     __START1__          1.0  capitalized      Thousands   \n",
       "1        capitalized      Thousands          1.0    lowercase             of   \n",
       "2          lowercase             of          1.0    lowercase  demonstrators   \n",
       "3          lowercase  demonstrators          1.0    lowercase           have   \n",
       "4          lowercase           have          1.0    lowercase        marched   \n",
       "...              ...            ...          ...          ...            ...   \n",
       "1050790    lowercase           said      47959.0    lowercase           they   \n",
       "1050791    lowercase           they      47959.0    lowercase      responded   \n",
       "1050792    lowercase      responded      47959.0    lowercase             to   \n",
       "1050793    lowercase             to      47959.0    lowercase            the   \n",
       "1050794    lowercase            the      47959.0    lowercase         attack   \n",
       "\n",
       "        tag  \n",
       "0         O  \n",
       "1         O  \n",
       "2         O  \n",
       "3         O  \n",
       "4         O  \n",
       "...      ..  \n",
       "1050790   O  \n",
       "1050791   O  \n",
       "1050792   O  \n",
       "1050793   O  \n",
       "1050794   O  \n",
       "\n",
       "[1050795 rows x 25 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"../data/ner.csv\", encoding='cp1252', on_bad_lines='warn')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a160a0ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2549.0     140\n",
       "11994.0    132\n",
       "608.0      124\n",
       "5805.0     122\n",
       "6344.0     120\n",
       "          ... \n",
       "37093.0      2\n",
       "8412.0       2\n",
       "39874.0      2\n",
       "40249.0      2\n",
       "38917.0      1\n",
       "Name: sentence_idx, Length: 35177, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sentence_idx.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "032a7b6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = data.sentence_idx.value_counts().index[:20000]\n",
    "len(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0fa44b59",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>lemma</th>\n",
       "      <th>next-lemma</th>\n",
       "      <th>next-next-lemma</th>\n",
       "      <th>next-next-pos</th>\n",
       "      <th>next-next-shape</th>\n",
       "      <th>next-next-word</th>\n",
       "      <th>next-pos</th>\n",
       "      <th>next-shape</th>\n",
       "      <th>next-word</th>\n",
       "      <th>...</th>\n",
       "      <th>prev-prev-lemma</th>\n",
       "      <th>prev-prev-pos</th>\n",
       "      <th>prev-prev-shape</th>\n",
       "      <th>prev-prev-word</th>\n",
       "      <th>prev-shape</th>\n",
       "      <th>prev-word</th>\n",
       "      <th>sentence_idx</th>\n",
       "      <th>shape</th>\n",
       "      <th>word</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>thousand</td>\n",
       "      <td>of</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>...</td>\n",
       "      <td>__start2__</td>\n",
       "      <td>__START2__</td>\n",
       "      <td>wildcard</td>\n",
       "      <td>__START2__</td>\n",
       "      <td>wildcard</td>\n",
       "      <td>__START1__</td>\n",
       "      <td>1.0</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>of</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>have</td>\n",
       "      <td>VBP</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>...</td>\n",
       "      <td>__start1__</td>\n",
       "      <td>__START1__</td>\n",
       "      <td>wildcard</td>\n",
       "      <td>__START1__</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>have</td>\n",
       "      <td>march</td>\n",
       "      <td>VBN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>marched</td>\n",
       "      <td>VBP</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>...</td>\n",
       "      <td>thousand</td>\n",
       "      <td>NNS</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>have</td>\n",
       "      <td>march</td>\n",
       "      <td>through</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>through</td>\n",
       "      <td>VBN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>marched</td>\n",
       "      <td>...</td>\n",
       "      <td>of</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>of</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>march</td>\n",
       "      <td>through</td>\n",
       "      <td>london</td>\n",
       "      <td>NNP</td>\n",
       "      <td>capitalized</td>\n",
       "      <td>London</td>\n",
       "      <td>IN</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>through</td>\n",
       "      <td>...</td>\n",
       "      <td>demonstr</td>\n",
       "      <td>NNS</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>have</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lowercase</td>\n",
       "      <td>marched</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0     lemma next-lemma next-next-lemma next-next-pos  \\\n",
       "0           0  thousand         of        demonstr           NNS   \n",
       "1           1        of   demonstr            have           VBP   \n",
       "2           2  demonstr       have           march           VBN   \n",
       "3           3      have      march         through            IN   \n",
       "4           4     march    through          london           NNP   \n",
       "\n",
       "  next-next-shape next-next-word next-pos next-shape      next-word  ...  \\\n",
       "0       lowercase  demonstrators       IN  lowercase             of  ...   \n",
       "1       lowercase           have      NNS  lowercase  demonstrators  ...   \n",
       "2       lowercase        marched      VBP  lowercase           have  ...   \n",
       "3       lowercase        through      VBN  lowercase        marched  ...   \n",
       "4     capitalized         London       IN  lowercase        through  ...   \n",
       "\n",
       "  prev-prev-lemma prev-prev-pos prev-prev-shape prev-prev-word   prev-shape  \\\n",
       "0      __start2__    __START2__        wildcard     __START2__     wildcard   \n",
       "1      __start1__    __START1__        wildcard     __START1__  capitalized   \n",
       "2        thousand           NNS     capitalized      Thousands    lowercase   \n",
       "3              of            IN       lowercase             of    lowercase   \n",
       "4        demonstr           NNS       lowercase  demonstrators    lowercase   \n",
       "\n",
       "       prev-word sentence_idx        shape           word tag  \n",
       "0     __START1__          1.0  capitalized      Thousands   O  \n",
       "1      Thousands          1.0    lowercase             of   O  \n",
       "2             of          1.0    lowercase  demonstrators   O  \n",
       "3  demonstrators          1.0    lowercase           have   O  \n",
       "4           have          1.0    lowercase        marched   O  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_data = data[data['sentence_idx'].isin(sentences)]\n",
    "new_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c90c3e73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "O        668694\n",
       "B-geo     28499\n",
       "B-org     15345\n",
       "B-tim     14932\n",
       "I-per     13417\n",
       "I-org     13218\n",
       "B-per     12708\n",
       "B-gpe     12464\n",
       "I-geo      5904\n",
       "I-tim      4765\n",
       "B-art       348\n",
       "B-eve       280\n",
       "I-eve       234\n",
       "I-art       226\n",
       "I-gpe       188\n",
       "B-nat       188\n",
       "I-nat        65\n",
       "Name: tag, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_data.tag.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1b4ab974",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['O', 'B-geo', 'B-org', 'B-tim', 'I-per', 'I-org', 'B-per', 'B-gpe',\n",
       "       'I-geo', 'I-tim', 'B-art', 'B-eve', 'I-eve', 'I-art', 'I-gpe', 'B-nat',\n",
       "       'I-nat'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = new_data.tag.value_counts().index\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cdd363b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'O': 0, 'B-geo': 1, 'B-org': 2, 'B-tim': 3, 'I-per': 4, 'I-org': 5, 'B-per': 6, 'B-gpe': 7, 'I-geo': 8, 'I-tim': 9, 'B-art': 10, 'B-eve': 11, 'I-eve': 12, 'I-art': 13, 'I-gpe': 14, 'B-nat': 15, 'I-nat': 16}\n"
     ]
    }
   ],
   "source": [
    "label_maps = {}\n",
    "c=0\n",
    "for k in labels:\n",
    "    label_maps[k] = c\n",
    "    c+=1\n",
    "    \n",
    "print(label_maps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0ffb60ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save label_map\n",
    "label_maps = json.dumps(label_maps)\n",
    "with open(\"label_maps.json\", 'w') as f:\n",
    "    f.write(label_maps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "31a9ca49",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read back label_maps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aacec909",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parsing data \n",
    "data_json = []\n",
    "for ind, i in enumerate(sentences):\n",
    "    ner_tags = []\n",
    "    tokens = []\n",
    "    sub_data = new_data[new_data['sentence_idx']==i]\n",
    "    for j in sub_data.index:\n",
    "        tokens.append(sub_data.loc[j,'word'])\n",
    "        ner_tag = label_maps[sub_data.loc[j,'tag']]\n",
    "        ner_tags.append(ner_tag)\n",
    "        \n",
    "    data = {'id':ind, 'ner_tags':ner_tags, 'tokens':tokens}\n",
    "    data_json.append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "05582aca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 5, 'ner_tags': [0, 0, 0, 0, 11, 12, 0, 3, 9, 9, 0, 0, 0, 0, 1, 8, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 1, 0, 0, 0, 0, 0, 6, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 12, 0, 3, 9, 9, 0, 0, 0, 0, 1, 8, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 1, 0, 0, 0, 0, 0, 6, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'tokens': ['After', 'failing', 'in', 'the', 'Korean', 'War', '(', '1950', '-', '53', ')', 'to', 'conquer', 'the', 'US-backed', 'Republic', 'of', 'Korea', '(', 'ROK', ')', 'in', 'the', 'southern', 'portion', 'by', 'force', ',', 'North', 'Korea', '(', 'DPRK', ')', ',', 'under', 'its', 'founder', 'President', 'KIM', 'Il', 'Sung', ',', 'adopted', 'a', 'policy', 'of', 'ostensible', 'diplomatic', 'and', 'economic', '\"', 'self-reliance', '\"', 'as', 'a', 'check', 'against', 'outside', 'influence', '.', 'After', 'failing', 'in', 'the', 'Korean', 'War', '(', '1950', '-', '53', ')', 'to', 'conquer', 'the', 'US-backed', 'Republic', 'of', 'Korea', '(', 'ROK', ')', 'in', 'the', 'southern', 'portion', 'by', 'force', ',', 'North', 'Korea', '(', 'DPRK', ')', ',', 'under', 'its', 'founder', 'President', 'KIM', 'Il', 'Sung', ',', 'adopted', 'a', 'policy', 'of', 'ostensible', 'diplomatic', 'and', 'economic', '\"', 'self-reliance', '\"', 'as', 'a', 'check', 'against', 'outside', 'influence', '.']}\n"
     ]
    }
   ],
   "source": [
    "print(data_json[5])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00f62358",
   "metadata": {},
   "source": [
    "### Train Val Test splitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "02f8e858",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_val_test_split(data, split_size=0.1):\n",
    "    random.shuffle(data)\n",
    "    total_size = len(data)\n",
    "    test_size = val_size = int(total_size*split_size)\n",
    "    test_indices = random.sample(range(total_size), test_size)\n",
    "    test_data = [ele for i, ele in enumerate(data) if i in test_indices]\n",
    "    train_data = [ele for i, ele in enumerate(data) if i not in test_indices]\n",
    "    total_size = len(train_data)\n",
    "    val_indices = random.sample(range(total_size), val_size)\n",
    "    val_data = [ele for i, ele in enumerate(train_data) if i in val_indices]\n",
    "    train_data = [ele for i, ele in enumerate(train_data) if i not in val_indices]\n",
    "    return train_data, val_data, test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d3364fa1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'id': 18077, 'ner_tags': [0, 0, 0, 0, 0, 6, 4, 4, 4, 0, 0, 0, 1, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'tokens': ['Unofficial', 'results', 'show', 'ruling', 'party', 'Prime', 'Minister', 'Jose', 'Ramos-Horta', 'and', 'parliament', 'chief', 'Francisco', '\"', 'Lu', 'Olo', '\"', 'Guterres', 'will', 'be', 'competing', 'in', 'a', 'runoff', 'election', '.']}, {'id': 16743, 'ner_tags': [0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0], 'tokens': ['The', 'attacks', 'came', 'as', 'negotiators', 'from', \"Shi'ite\", 'and', 'Kurdish', 'factions', 'that', 'dominate', 'parliament', 'continued', 'talks', 'aimed', 'at', 'winning', 'last-minute', 'Sunni', 'Arab', 'backing', 'for', 'the', 'draft', 'constitution', '.']}, {'id': 13280, 'ner_tags': [6, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'tokens': ['President', 'Mesa', 'submitted', 'his', 'resignation', 'Monday', ',', 'saying', 'he', 'could', 'no', 'longer', 'lead', 'the', 'poor', 'Andean', 'nation', 'in', 'the', 'face', 'of', 'continuing', 'large', 'demonstrations', 'over', 'his', 'government', \"'s\", 'policies', '.']}]\n"
     ]
    }
   ],
   "source": [
    "train, val, test = train_val_test_split(data_json, split_size=0.1)\n",
    "print(train[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e6cfee2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# json_dump\n",
    "train = json.dumps(train)\n",
    "val = json.dumps(val)\n",
    "test = json.dumps(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "80c06224",
   "metadata": {},
   "outputs": [],
   "source": [
    "for file, d  in zip(['train', 'val', 'test'], [train, val, test]):\n",
    "    with open(f\"../data/huggingface/{file}.json\", 'w') as f:\n",
    "        f.write(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "31c12d9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read back data\n",
    "with open(f\"../data/huggingface/train.json\", 'r') as f:\n",
    "    train = json.load(f)\n",
    "with open(f\"../data/huggingface/val.json\", 'r') as f:\n",
    "    val = json.load(f)\n",
    "with open(f\"../data/huggingface/test.json\", 'r') as f:\n",
    "    test = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c42aa892",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 13280, 'ner_tags': [6, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'tokens': ['President', 'Mesa', 'submitted', 'his', 'resignation', 'Monday', ',', 'saying', 'he', 'could', 'no', 'longer', 'lead', 'the', 'poor', 'Andean', 'nation', 'in', 'the', 'face', 'of', 'continuing', 'large', 'demonstrations', 'over', 'his', 'government', \"'s\", 'policies', '.']}\n"
     ]
    }
   ],
   "source": [
    "print(train[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "409448d6",
   "metadata": {},
   "source": [
    "### Inspecting required dataset format by dowlodaing and loading a dummy dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ae4b5ceb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset wnut_17 (C:/Users/ritap/.cache/huggingface/datasets/wnut_17/wnut_17/1.0.0/077c7f08b8dbc800692e8c9186cdf3606d5849ab0e7be662e6135bb10eba54f9)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "154b81e571454948bfe1374588535d82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'tokens', 'ner_tags'],\n",
       "        num_rows: 3394\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'tokens', 'ner_tags'],\n",
       "        num_rows: 1009\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'tokens', 'ner_tags'],\n",
       "        num_rows: 1287\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "wnut = load_dataset(\"wnut_17\")\n",
    "wnut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "928fd2fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['id', 'tokens', 'ner_tags'],\n",
      "    num_rows: 3394\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "print(wnut[\"train\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cb1f9106",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': '2', 'tokens': ['Pxleyes', 'Top', '50', 'Photography', 'Contest', 'Pictures', 'of', 'August', '2010', '...', 'http://bit.ly/bgCyZ0', '#photography'], 'ner_tags': [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}\n"
     ]
    }
   ],
   "source": [
    "print(wnut[\"train\"][2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6a95c38",
   "metadata": {},
   "source": [
    "### Our dataset is a list of dictionaries, so we have to use Dataset.from_list(data) for typecasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ce187592",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c62ee9e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['_TF_DATASET_REFS', '__class__', '__del__', '__delattr__', '__dict__', '__dir__', '__doc__', '__enter__', '__eq__', '__exit__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__getitems__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_build_local_temp_path', '_check_index_is_initialized', '_estimate_nbytes', '_generate_examples_from_shards', '_get_cache_file_path', '_get_output_signature', '_getitem', '_map_single', '_new_dataset_with_indices', '_push_parquet_shards_to_hub', '_save_to_disk_single', '_select_contiguous', '_select_with_indices_mapping', 'add_column', 'add_elasticsearch_index', 'add_faiss_index', 'add_faiss_index_from_external_arrays', 'add_item', 'align_labels_with_mapping', 'builder_name', 'cache_files', 'cast', 'cast_column', 'citation', 'class_encode_column', 'cleanup_cache_files', 'column_names', 'config_name', 'data', 'dataset_size', 'description', 'download_checksums', 'download_size', 'drop_index', 'export', 'features', 'filter', 'flatten', 'flatten_indices', 'format', 'formatted_as', 'from_buffer', 'from_csv', 'from_dict', 'from_file', 'from_generator', 'from_json', 'from_list', 'from_pandas', 'from_parquet', 'from_sql', 'from_text', 'get_index', 'get_nearest_examples', 'get_nearest_examples_batch', 'homepage', 'info', 'is_index_initialized', 'iter', 'license', 'list_indexes', 'load_elasticsearch_index', 'load_faiss_index', 'load_from_disk', 'map', 'num_columns', 'num_rows', 'prepare_for_task', 'push_to_hub', 'remove_columns', 'rename_column', 'rename_columns', 'reset_format', 'save_faiss_index', 'save_to_disk', 'search', 'search_batch', 'select', 'select_columns', 'set_format', 'set_transform', 'shape', 'shard', 'shuffle', 'size_in_bytes', 'sort', 'split', 'supervised_keys', 'task_templates', 'to_csv', 'to_dict', 'to_iterable_dataset', 'to_json', 'to_list', 'to_pandas', 'to_parquet', 'to_sql', 'to_tf_dataset', 'train_test_split', 'unique', 'version', 'with_format', 'with_transform']\n"
     ]
    }
   ],
   "source": [
    "print(dir(Dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1d764606",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['id', 'ner_tags', 'tokens'],\n",
      "    num_rows: 16000\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "train = Dataset.from_list(train)\n",
    "test = Dataset.from_list(test)\n",
    "print(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5eb498ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 13280, 'ner_tags': [6, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'tokens': ['President', 'Mesa', 'submitted', 'his', 'resignation', 'Monday', ',', 'saying', 'he', 'could', 'no', 'longer', 'lead', 'the', 'poor', 'Andean', 'nation', 'in', 'the', 'face', 'of', 'continuing', 'large', 'demonstrations', 'over', 'his', 'government', \"'s\", 'policies', '.']}\n"
     ]
    }
   ],
   "source": [
    "print(train[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "446630d8",
   "metadata": {},
   "source": [
    "### Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d68a0cbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\") # use the same tokenizer as our model architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2a7a792d",
   "metadata": {},
   "outputs": [],
   "source": [
    "example = train[2] # taking a random example for inspection\n",
    "tokenized_input = tokenizer(example[\"tokens\"], is_split_into_words=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "7cde48e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[CLS]', 'president', 'mesa', 'submitted', 'his', 'resignation', 'monday', ',', 'saying', 'he', 'could', 'no', 'longer', 'lead', 'the', 'poor', 'and', '##ean', 'nation', 'in', 'the', 'face', 'of', 'continuing', 'large', 'demonstrations', 'over', 'his', 'government', \"'\", 's', 'policies', '.', '[SEP]']\n"
     ]
    }
   ],
   "source": [
    "tokens = tokenizer.convert_ids_to_tokens(tokenized_input[\"input_ids\"])\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02b57ff8",
   "metadata": {},
   "source": [
    "### As mentioned in huggingface documentation, I am pasting exactly what they said:-\n",
    "##### However, this adds some special tokens [CLS] and [SEP] and the subword tokenization creates a mismatch between the input and labels. A single word corresponding to a single label may now be split into two subwords. You’ll need to realign the tokens and labels by:\n",
    "\n",
    "##### Mapping all tokens to their corresponding word with the word_ids method.\n",
    "##### Assigning the label -100 to the special tokens [CLS] and [SEP] so they’re ignored by the PyTorch loss function.\n",
    "##### Only labeling the first token of a given word. Assign -100 to other subtokens from the same word.\n",
    "##### Here is how you can create a function to realign the tokens and labels, and truncate sequences to be no longer than DistilBERT’s maximum input length:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ae52a362",
   "metadata": {},
   "outputs": [],
   "source": [
    "# this function does the above\n",
    "def tokenize_and_align_labels(examples):\n",
    "    tokenized_inputs = tokenizer(examples[\"tokens\"], truncation=True, is_split_into_words=True)\n",
    "\n",
    "    labels = []\n",
    "    for i, label in enumerate(examples[f\"ner_tags\"]):\n",
    "        word_ids = tokenized_inputs.word_ids(batch_index=i)  # Map tokens to their respective word.\n",
    "        previous_word_idx = None\n",
    "        label_ids = []\n",
    "        for word_idx in word_ids:  # Set the special tokens to -100.\n",
    "            if word_idx is None:\n",
    "                label_ids.append(-100)\n",
    "            elif word_idx != previous_word_idx:  # Only label the first token of a given word.\n",
    "                label_ids.append(label[word_idx])\n",
    "            else:\n",
    "                label_ids.append(-100)\n",
    "            previous_word_idx = word_idx\n",
    "        labels.append(label_ids)\n",
    "\n",
    "    tokenized_inputs[\"labels\"] = labels\n",
    "    return tokenized_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6bdd4001",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/16000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/2000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# mapping \n",
    "train_tokenized = train.map(tokenize_and_align_labels, batched=True)\n",
    "test_tokenized = test.map(tokenize_and_align_labels, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c812f3e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['id', 'ner_tags', 'tokens', 'input_ids', 'attention_mask', 'labels'],\n",
      "    num_rows: 16000\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "print(train_tokenized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e993c690",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 13280, 'ner_tags': [6, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'tokens': ['President', 'Mesa', 'submitted', 'his', 'resignation', 'Monday', ',', 'saying', 'he', 'could', 'no', 'longer', 'lead', 'the', 'poor', 'Andean', 'nation', 'in', 'the', 'face', 'of', 'continuing', 'large', 'demonstrations', 'over', 'his', 'government', \"'s\", 'policies', '.'], 'input_ids': [101, 2343, 15797, 7864, 2010, 8172, 6928, 1010, 3038, 2002, 2071, 2053, 2936, 2599, 1996, 3532, 1998, 11219, 3842, 1999, 1996, 2227, 1997, 5719, 2312, 13616, 2058, 2010, 2231, 1005, 1055, 6043, 1012, 102], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, 6, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -100, 0, 0, -100]}\n"
     ]
    }
   ],
   "source": [
    "print(train_tokenized[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b4d9ce3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save as pikle file\n",
    "import pickle\n",
    "with open('../data/huggingface/train_tokenized.pkl', 'wb') as file:      \n",
    "    pickle.dump(train_tokenized, file)\n",
    "with open('../data/huggingface/test_tokenized.pkl', 'wb') as file:      \n",
    "    pickle.dump(test_tokenized, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c35fe6a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "7a175bc1",
   "metadata": {},
   "source": [
    "## Rough work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "9c19c1c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    def word_ids(self, batch_index: int = 0) -> List[Optional[int]]:\n",
      "        \"\"\"\n",
      "        Return a list mapping the tokens to their actual word in the initial sentence for a fast tokenizer.\n",
      "\n",
      "        Args:\n",
      "            batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.\n",
      "\n",
      "        Returns:\n",
      "            `List[Optional[int]]`: A list indicating the word corresponding to each token. Special tokens added by the\n",
      "            tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding word\n",
      "            (several tokens will be mapped to the same word index if they are parts of that word).\n",
      "        \"\"\"\n",
      "        if not self._encodings:\n",
      "            raise ValueError(\n",
      "                \"word_ids() is not available when using non-fast tokenizers (e.g. instance of a `XxxTokenizerFast`\"\n",
      "                \" class).\"\n",
      "            )\n",
      "        return self._encodings[batch_index].word_ids\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "930"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import inspect as i\n",
    "import sys\n",
    "sys.stdout.write(i.getsource(tokenized_inputs.word_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "6316e638",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    def word_to_tokens(\n",
      "        self, batch_or_word_index: int, word_index: Optional[int] = None, sequence_index: int = 0\n",
      "    ) -> Optional[TokenSpan]:\n",
      "        \"\"\"\n",
      "        Get the encoded token span corresponding to a word in a sequence of the batch.\n",
      "\n",
      "        Token spans are returned as a [`~tokenization_utils_base.TokenSpan`] with:\n",
      "\n",
      "        - **start** -- Index of the first token.\n",
      "        - **end** -- Index of the token following the last token.\n",
      "\n",
      "        Can be called as:\n",
      "\n",
      "        - `self.word_to_tokens(word_index, sequence_index: int = 0)` if batch size is 1\n",
      "        - `self.word_to_tokens(batch_index, word_index, sequence_index: int = 0)` if batch size is greater or equal to\n",
      "          1\n",
      "\n",
      "        This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words\n",
      "        are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized\n",
      "        words.\n",
      "\n",
      "        Args:\n",
      "            batch_or_word_index (`int`):\n",
      "                Index of the sequence in the batch. If the batch only comprises one sequence, this can be the index of\n",
      "                the word in the sequence.\n",
      "            word_index (`int`, *optional*):\n",
      "                If a batch index is provided in *batch_or_token_index*, this can be the index of the word in the\n",
      "                sequence.\n",
      "            sequence_index (`int`, *optional*, defaults to 0):\n",
      "                If pair of sequences are encoded in the batch this can be used to specify which sequence in the pair (0\n",
      "                or 1) the provided word index belongs to.\n",
      "\n",
      "        Returns:\n",
      "            ([`~tokenization_utils_base.TokenSpan`], *optional*): Span of tokens in the encoded sequence. Returns\n",
      "            `None` if no tokens correspond to the word. This can happen especially when the token is a special token\n",
      "            that has been used to format the tokenization. For example when we add a class token at the very beginning\n",
      "            of the tokenization.\n",
      "        \"\"\"\n",
      "\n",
      "        if not self._encodings:\n",
      "            raise ValueError(\"word_to_tokens() is not available when using Python based tokenizers\")\n",
      "        if word_index is not None:\n",
      "            batch_index = batch_or_word_index\n",
      "        else:\n",
      "            batch_index = 0\n",
      "            word_index = batch_or_word_index\n",
      "        if batch_index < 0:\n",
      "            batch_index = self._batch_size + batch_index\n",
      "        if word_index < 0:\n",
      "            word_index = self._seq_len + word_index\n",
      "        span = self._encodings[batch_index].word_to_tokens(word_index, sequence_index)\n",
      "        return TokenSpan(*span) if span is not None else None\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2619"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys.stdout.write(i.getsource(tokenized_inputs.word_to_tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "7a9c6af9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[CLS]', '@', 'paul', '##walk', 'it', \"'\", 's', 'the', 'view', 'from', 'where', 'i', \"'\", 'm', 'living', 'for', 'two', 'weeks', '.', 'empire', 'state', 'building', '=', 'es', '##b', '.', 'pretty', 'bad', 'storm', 'here', 'last', 'evening', '.', '[SEP]']\n"
     ]
    }
   ],
   "source": [
    "example = wnut[\"train\"][0]\n",
    "tokenized_input = tokenizer(example[\"tokens\"], is_split_into_words=True)\n",
    "tokens = tokenizer.convert_ids_to_tokens(tokenized_input[\"input_ids\"])\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "c46a1c42",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['_MutableMapping__marker', '__abstractmethods__', '__class__', '__contains__', '__copy__', '__delattr__', '__delitem__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getitem__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__reversed__', '__setattr__', '__setitem__', '__setstate__', '__sizeof__', '__slots__', '__str__', '__subclasshook__', '__weakref__', '_abc_impl', '_encodings', '_n_sequences', 'char_to_token', 'char_to_word', 'clear', 'convert_to_tensors', 'copy', 'data', 'encodings', 'fromkeys', 'get', 'is_fast', 'items', 'keys', 'n_sequences', 'pop', 'popitem', 'sequence_ids', 'setdefault', 'to', 'token_to_chars', 'token_to_sequence', 'token_to_word', 'tokens', 'update', 'values', 'word_ids', 'word_to_chars', 'word_to_tokens', 'words']\n"
     ]
    }
   ],
   "source": [
    "print(dir(tokenized_input))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd486ed4",
   "metadata": {},
   "outputs": [],
   "source": [
    " "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}