predict_relations_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/romainbourgeois/miniconda3/lib/python3.8/site-packages/jax/_src/lib/__init__.py:34: UserWarning: JAX on Mac ARM machines is experimental and minimally tested. Please see https://github.com/google/jax/issues/5501 in the event of problems.\n",
      "  warnings.warn(\"JAX on Mac ARM machines is experimental and minimally tested. \"\n",
      "Since the GPL-licensed package `unidecode` is not installed, using Python's `unicodedata` package which yields worse results.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import spacy\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torchimpl.src.misc import save_as_pickle, load_pickle, get_subject_objects\n",
    "from tqdm import tqdm\n",
    "import logging\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from cleantext import clean\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, DataCollatorForTokenClassification, AutoModelForTokenClassification, TrainingArguments, Trainer, AutoModelForMaskedLM\n",
    "import datasets\n",
    "from datasets import Features, Value, Sequence, load_metric, load_dataset\n",
    "from datasets import Dataset\n",
    "import torch\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.cluster import KMeans\n",
    "import random\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# preprocessing functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_sent(sent):\n",
    "    if sent not in [\" \", \"\\n\", \"\"]:\n",
    "        sent = sent.strip(\"\\n\")            \n",
    "        sent = re.sub('<[A-Z]+/*>', '', sent) # remove special tokens eg. <FIL/>, <S>\n",
    "        sent = re.sub(r\"[\\*\\\"\\n\\\\…\\+\\-\\/\\=\\(\\)‘•€\\[\\]\\|♫:;—”“~`#]\", \" \", sent)\n",
    "        sent = re.sub(' {2,}', ' ', sent) # remove extra spaces > 1\n",
    "        sent = re.sub(\"^ +\", \"\", sent) # remove space in front\n",
    "        sent = re.sub(r\"([\\.\\?,!]){2,}\", r\"\\1\", sent) # remove multiple puncs\n",
    "        sent = re.sub(r\" +([\\.\\?,!])\", r\"\\1\", sent) # remove extra spaces in front of punc\n",
    "        sent = re.sub(r\"([A-Z]{2,})\", lambda x: x.group(1).capitalize(), sent) # Replace all CAPS with capitalize\n",
    "        sent=sent.replace(\"?\",\"\")\n",
    "        sent=sent.replace(\"@\",\"\")\n",
    "        sent=sent.replace(\"®\",\"\")\n",
    "        return sent\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def contractions(phrase):\n",
    "    phrase = re.sub(r\"won\\'t\", \"will not\", phrase) # 's could mean possession\n",
    "    phrase = re.sub(r\"won't\", \"will not\", phrase)  \n",
    "    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n",
    "    phrase = re.sub(r\"can't\", \"can not\", phrase)\n",
    "    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"n't\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
    "    phrase = re.sub(r\"'re\", \" are\", phrase)\n",
    "    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
    "    phrase = re.sub(r\"'ll\", \" will\", phrase)\n",
    "    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"'t\", \" not\", phrase)\n",
    "    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
    "    phrase = re.sub(r\"'ve\", \" have\", phrase)\n",
    "    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
    "    phrase = re.sub(r\"'m\", \" am\", phrase)\n",
    "    phrase = re.sub(r\"wont\", \"will not\", phrase)\n",
    "    phrase = re.sub(r\"dont\", \"do not\", phrase)\n",
    "    phrase = re.sub(r\"werent\", \"were not\", phrase)\n",
    "    phrase = re.sub(r\"'m\", \" am\", phrase)\n",
    "\n",
    "    return phrase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleanfunc(t):\n",
    "    return clean(t,\n",
    "    fix_unicode=True,               # fix various unicode errors\n",
    "    to_ascii=True,                  # transliterate to closest ASCII representation\n",
    "    lower=False,  #if YES lowercase targets            # lowercase text\n",
    "    no_line_breaks=False,           # fully strip line breaks as opposed to only normalizing them\n",
    "    no_urls=True,                  # replace all URLs with a special token\n",
    "    no_emails=True,                # replace all email addresses with a special token\n",
    "    no_phone_numbers=True,         # replace all phone numbers with a special token\n",
    "    no_numbers=False,               # replace all numbers with a special token\n",
    "    no_digits=False,                # replace all digits with a special token\n",
    "    no_currency_symbols=False,      # replace all currency symbols with a special token\n",
    "    no_punct=False,                 # remove punctuations\n",
    "    replace_with_punct=\"\",          # instead of removing punctuations you may replace them\n",
    "    replace_with_url=\"<URL>\",\n",
    "    replace_with_email=\"<EMAIL>\",\n",
    "    replace_with_phone_number=\"<PHONE>\",\n",
    "    replace_with_number=\"<NUMBER>\",\n",
    "    replace_with_digit=\"0\",\n",
    "    replace_with_currency_symbol=\"<CUR>\",\n",
    "    lang=\"en\"                       \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NER pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
    "model_loaded = AutoModelForTokenClassification.from_pretrained(os.getcwd()+\"/ner/distilbert_merged_.pt\",local_files_only=True)\n",
    "label_list=['O', 'B-ORG','I-ORG','B-PSM','I-PSM','B-AMNT','I-AMNT','B-PSN','I-PSN','B-LOC','I-LOC']\n",
    "trainer = Trainer(\n",
    "      model=model_loaded,\n",
    "      tokenizer=tokenizer,\n",
    "  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_sents(text): # has to be strings (handles multi-sentence strings)\n",
    "    r={}\n",
    "    r['text']=[]\n",
    "    i=0\n",
    "    text=process_sent(contractions(cleanfunc(text)))\n",
    "    split=text.split('\\n')\n",
    "    for ss in split:\n",
    "        sents=nltk.sent_tokenize(ss)\n",
    "        for s_ in sents:\n",
    "            r['text'].append(s_)\n",
    "    return pd.DataFrame(r)\n",
    "\n",
    "def tokenize_(examples): # examples has to be a list\n",
    "    tokenized_inputs = tokenizer(examples[\"text\"], truncation=True, is_split_into_words=False, max_length=385,padding=\"max_length\")\n",
    "    return tokenized_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "def idx_inarow(predictions): # get predicted entity from tokenized sequence\n",
    "    ids_idxs=[]\n",
    "    for si in range(predictions.shape[0]):\n",
    "        ids_idx=[]\n",
    "        i=1\n",
    "        for z in range(len(predictions[si])): #loop necessary to fix start and end token mismatch\n",
    "            if predictions[si][z]>0:\n",
    "                if predictions[si][z]%2==1:\n",
    "                    predictions[si][z]=predictions[si][z]+1\n",
    "        while i<len(predictions[si])-1:\n",
    "            if predictions[si][i]!=0:\n",
    "                ii=i\n",
    "                i=i+1\n",
    "                j=i\n",
    "                while predictions[si][ii]==predictions[si][j]:\n",
    "                    j=j+1\n",
    "                    i=i+1\n",
    "                    if j>=len(predictions[si]):\n",
    "                        break\n",
    "                ids_idx.append([ii,j])\n",
    "            else:\n",
    "                i=i+1\n",
    "        ids_idxs.append(ids_idx)\n",
    "    return ids_idxs\n",
    "\n",
    "def seq_withEnts(ds,predictions):   # outputs sequence with list of predicted entities\n",
    "    to_pred=[]\n",
    "    inarow=idx_inarow(predictions) # compute ents indexes\n",
    "    for s in range(len(ds)): # sentence loop\n",
    "        text=ds[s]['text'].lower()\n",
    "        ents=[]\n",
    "        for id in inarow[s]:\n",
    "            if tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(ds[s]['input_ids'][id[0]:id[1]])) in text: # translate sequence of token ids to string\n",
    "                ents.append(tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(ds[s]['input_ids'][id[0]:id[1]])))\n",
    "        uniqueents=list(set(ents)) # get unique list of entities\n",
    "        to_pred.append([text,uniqueents])\n",
    "    return to_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_idxs(e,s): # find index range of entity \n",
    "    spec=['$','¥','€','£','(',')'] # deals with characters that do not work with the re.search method\n",
    "    ss=s\n",
    "    c=0\n",
    "    presence=False\n",
    "    for i in e:\n",
    "        if i in spec:\n",
    "            c=c+1 \n",
    "            presence=True     \n",
    "    if presence==True:\n",
    "        for ii in spec:\n",
    "            ss=ss.replace(ii,'')\n",
    "            e=e.replace(ii,'')\n",
    "        idx=re.search(e,ss).span()\n",
    "        idx_=[idx[0],idx[0]+c]  \n",
    "    else:\n",
    "        idx_=re.search(e,ss).span()       \n",
    "    return idx_\n",
    "\n",
    "def combination(l): # combine all possible matches of entities\n",
    "    ll=[]\n",
    "    for i in range(len(l)):\n",
    "        for j in range(len(l)):\n",
    "            if j==i:\n",
    "                continue\n",
    "            else:\n",
    "                ll.append([l[i],l[j]])\n",
    "    return ll\n",
    "\n",
    "def create_combinedSeqs(seqs): # add special tokens to the relation candidates\n",
    "    data=[]\n",
    "    for s in seqs:\n",
    "        text=s[0]\n",
    "        combs=combination(s[1])\n",
    "        d=[]\n",
    "        for c in combs:\n",
    "            dd={}\n",
    "            idx1=search_idxs(c[0],text)\n",
    "            idx2=search_idxs(c[1],text)\n",
    "            if idx1[0]<idx2[0]:\n",
    "                input=text[:idx1[0]]+\"[E1] \"+c[0]+\" [/E1] \"+text[idx1[1]:idx2[0]]+\"[E2] \"+c[1]+\" [/E2] \"+text[idx2[1]:]\n",
    "                dd['text']=[text]\n",
    "                dd['inputs']=[input]\n",
    "                dd['headFirst']=[True]\n",
    "                dd[\"head\"]=[c[0]]\n",
    "                dd[\"child\"]=[c[1]]\n",
    "                d.append(dd)\n",
    "            else:\n",
    "                input=text[:idx2[0]]+\"[E2] \"+c[1]+\" [/E2] \"+text[idx2[1]:idx1[0]]+\"[E1] \"+c[0]+\" [/E1] \"+text[idx1[1]:]\n",
    "                dd['text']=[text]\n",
    "                dd['inputs']=[input]\n",
    "                dd['headFirst']=[False]\n",
    "                dd[\"head\"]=[c[1]]\n",
    "                dd[\"child\"]=[c[0]]\n",
    "                d.append(dd)\n",
    "        data.append(d)\n",
    "    return data\n",
    "         "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train relation classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-fb6f12f54e4b24af\n",
      "Reusing dataset json (/Users/romainbourgeois/.cache/huggingface/datasets/json/default-fb6f12f54e4b24af/0.0.0/ac0ca5f5289a6cf108e706efcf040422dbbfa8e658dee6a819f20d76bb84d26b)\n",
      "100%|██████████| 2/2 [00:00<00:00, 689.46it/s]\n",
      "loading file /Users/romainbourgeois/Desktop/matching_the_blanks/tokenizer_fine_tuned_.pt/vocab.txt\n",
      "loading file /Users/romainbourgeois/Desktop/matching_the_blanks/tokenizer_fine_tuned_.pt/tokenizer.json\n",
      "loading file /Users/romainbourgeois/Desktop/matching_the_blanks/tokenizer_fine_tuned_.pt/added_tokens.json\n",
      "loading file /Users/romainbourgeois/Desktop/matching_the_blanks/tokenizer_fine_tuned_.pt/special_tokens_map.json\n",
      "loading file /Users/romainbourgeois/Desktop/matching_the_blanks/tokenizer_fine_tuned_.pt/tokenizer_config.json\n",
      "loading configuration file /Users/romainbourgeois/Desktop/matching_the_blanks/fine_tuned_.pt/config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"yiyanghkust/finbert-pretrain\",\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"output_hidden_states\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.11.3\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30877\n",
      "}\n",
      "\n",
      "loading weights file /Users/romainbourgeois/Desktop/matching_the_blanks/fine_tuned_.pt/pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing BertForMaskedLM.\n",
      "\n",
      "All the weights of BertForMaskedLM were initialized from the model checkpoint at /Users/romainbourgeois/Desktop/matching_the_blanks/fine_tuned_.pt.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use BertForMaskedLM for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "# import fine-tuned model and tokenizer, prepare dataset for outputing embeddings from hidden layers (data to be sent to the classifier)\n",
    "dataset = load_dataset('json',data_files={'train':'labelledData/reldatatrain.json', 'test': 'labelledData/reldatatest.json'}, field='data')\n",
    "tokenizer_ = AutoTokenizer.from_pretrained(os.getcwd()+\"/tokenizer_fine_tuned_.pt\",local_files_only=True)\n",
    "model_loaded_ = AutoModelForMaskedLM.from_pretrained(os.getcwd()+\"/fine_tuned_.pt\",local_files_only=True, output_hidden_states=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=['0','PARTNERSHIP','RESEARCH_PROJECT','SUBSIDIARY','PURCHASE','FINANCING','RECRUITMENT','LAUNCH_PRODUCT-SERVICE','HAS_PRODUCT-SERVICE',\n",
    "'OPERATES_IN_MARKET','BASED_IN','WORKS_IN']\n",
    "\n",
    "def firsttoken(output):\n",
    "    return output[0]\n",
    "\n",
    "def EntTokens(output, reverse, e11, e22):\n",
    "    if reverse==True:\n",
    "        return np.concatenate((output[e11],output[e22]),axis=0)\n",
    "    else:\n",
    "        return np.concatenate((output[e22],output[e11]),axis=0)\n",
    "\n",
    "def maxpool_ents(output, reverse, e11, _e11, e22, _e22):\n",
    "    ee1=output[e11+1:_e11]\n",
    "    if ee1.shape[0]==0:\n",
    "        ee1=np.zeros(768)\n",
    "    elif ee1.shape[0]>1:\n",
    "        ee1=ee1.max(axis=0)\n",
    "    else:\n",
    "        ee1=ee1[0]\n",
    "    ee2=output[e22+1:_e22]\n",
    "    if ee2.shape[0]==0:\n",
    "        ee2=np.zeros(768)\n",
    "    elif ee2.shape[0]>1:\n",
    "        ee2=ee2.max(axis=0)\n",
    "    else:\n",
    "        ee2=ee2[0]\n",
    "    if reverse==True:\n",
    "        return np.concatenate((ee1,ee2),axis=0)\n",
    "    else:\n",
    "        return np.concatenate((ee1,ee2),axis=0)\n",
    "\n",
    "def data(dataset, e1=30873, _e1=30875, e2=30874, _e2=30876): \n",
    "    inp=tokenizer_(dataset[\"inputs\"], is_split_into_words=True, truncation=True, max_length=512, padding='max_length')\n",
    "    outputs=model_loaded_(torch.tensor(inp['input_ids']).reshape(1,-1))\n",
    "    output=outputs[-1][-1].detach().numpy()[0,:,:]\n",
    "    reverse=dataset['head_first']\n",
    "    label=dataset['label']\n",
    "    e11=inp['input_ids'].index(e1)\n",
    "    _e11=inp['input_ids'].index(_e1)\n",
    "    e22=inp['input_ids'].index(e2)\n",
    "    _e22=inp['input_ids'].index(_e2)\n",
    "    l=labels.index(label)\n",
    "    firsttokendata=firsttoken(output)\n",
    "    EntTokensdata=EntTokens(output, reverse, e11, e22)\n",
    "    maxpool_entsdata=maxpool_ents(output, reverse,e11,_e11,e22,_e22)\n",
    "    return l,firsttokendata,EntTokensdata,maxpool_entsdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train=np.array(data(dataset['train'][0])[0]).reshape(1,-1)\n",
    "X_train1=data(dataset['train'][0])[1].reshape(1,-1)\n",
    "X_train2=data(dataset['train'][0])[2].reshape(1,-1)\n",
    "X_train3=data(dataset['train'][0])[3].reshape(1,-1)\n",
    "\n",
    "for i in range(len(dataset['train'])):\n",
    "    o,one,two,three=data(dataset['train'][i])\n",
    "    y_train=np.concatenate((y_train,np.array(o).reshape(1,-1)),axis=0)\n",
    "    X_train1=np.concatenate((X_train1,one.reshape(1,-1)),axis=0)\n",
    "    X_train2=np.concatenate((X_train2,two.reshape(1,-1)),axis=0)\n",
    "    X_train3=np.concatenate((X_train3,three.reshape(1,-1)),axis=0)\n",
    "\n",
    "y_test=np.array(data(dataset['test'][0])[0]).reshape(1,-1)\n",
    "X_test1=data(dataset['test'][0])[1].reshape(1,-1)\n",
    "X_test2=data(dataset['test'][0])[2].reshape(1,-1)\n",
    "X_test3=data(dataset['test'][0])[3].reshape(1,-1)\n",
    "\n",
    "for i in range(len(dataset['test'])):\n",
    "    o,one,two,three=data(dataset['test'][i])\n",
    "    y_test=np.concatenate((y_test,np.array(o).reshape(1,-1)),axis=0)\n",
    "    X_test1=np.concatenate((X_test1,one.reshape(1,-1)),axis=0)\n",
    "    X_test2=np.concatenate((X_test2,two.reshape(1,-1)),axis=0)\n",
    "    X_test3=np.concatenate((X_test3,three.reshape(1,-1)),axis=0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "choosing nuber of nearest neighbors and selecting method 2 for feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/romainbourgeois/miniconda3/lib/python3.8/site-packages/sklearn/neighbors/_classification.py:200: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  return self._fit(X, y)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.87915937 0.10909091 0.         0.4        0.         0.\n",
      " 0.18604651 0.11428571 0.26666667 0.16666667]\n",
      "0.13808405203754043\n"
     ]
    }
   ],
   "source": [
    "neigh = KNeighborsClassifier(n_neighbors=2)\n",
    "neigh.fit(X_train2, y_train)\n",
    "y_pred=neigh.predict(X_test2)\n",
    "print(f1_score(y_test, y_pred, average=None))\n",
    "print(f1_score(y_test, y_pred, average=None)[1:].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/romainbourgeois/miniconda3/lib/python3.8/site-packages/sklearn/neighbors/_classification.py:200: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  return self._fit(X, y)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.88734835 0.08823529 0.         0.4        0.         0.\n",
      " 0.16216216 0.20512821 0.23529412 0.16666667]\n",
      "0.13972071619130444\n"
     ]
    }
   ],
   "source": [
    "neigh = KNeighborsClassifier(n_neighbors=4)\n",
    "neigh.fit(X_train2, y_train)\n",
    "y_pred=neigh.predict(X_test2)\n",
    "print(f1_score(y_test, y_pred, average=None))\n",
    "print(f1_score(y_test, y_pred, average=None)[1:].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/romainbourgeois/miniconda3/lib/python3.8/site-packages/sklearn/neighbors/_classification.py:200: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  return self._fit(X, y)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.86163142 0.17857143 0.         0.5        0.         0.\n",
      " 0.17142857 0.15789474 0.35294118 0.15384615]\n",
      "0.16829800746209417\n"
     ]
    }
   ],
   "source": [
    "neigh = KNeighborsClassifier(n_neighbors=3)\n",
    "neigh.fit(X_train2, y_train)\n",
    "y_pred=neigh.predict(X_test2)\n",
    "print(f1_score(y_test, y_pred, average=None))\n",
    "print(f1_score(y_test, y_pred, average=None)[1:].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# predicted random sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_pred(test_, label, e1=30873, _e1=30875, e2=30874, _e2=30876):\n",
    "    features=Features({'text': datasets.Sequence(datasets.Value(\"string\"))})\n",
    "    dataset = Dataset.from_pandas(make_sents(test_))\n",
    "    ds=dataset.map(tokenize_)\n",
    "    predictions, labels, _=trainer.predict(ds)\n",
    "    predictions = np.argmax(predictions, axis=2)\n",
    "    seq_ents=seq_withEnts(ds,predictions)\n",
    "    print(seq_ents)\n",
    "    data_=create_combinedSeqs(seq_ents)\n",
    "    rel=[]\n",
    "    for i in data_:\n",
    "        print(\"predicting sentence :\",i[0]['text'])\n",
    "        print(\"predicting on \",len(i),\" different combinations of entities\")\n",
    "        for j in i:\n",
    "            datast=Dataset.from_dict(j)\n",
    "            inp=tokenizer_(datast[\"inputs\"], is_split_into_words=False, truncation=True, max_length=512, padding='max_length')\n",
    "            outputs=model_loaded_(torch.tensor(inp['input_ids']).reshape(1,-1))\n",
    "            output=outputs[-1][-1].detach().numpy()[0,:,:]\n",
    "            reverse=datast['headFirst']\n",
    "            e11=inp['input_ids'][0].index(e1)\n",
    "            _e11=inp['input_ids'][0].index(_e1)\n",
    "            e22=inp['input_ids'][0].index(e2)\n",
    "            _e22=inp['input_ids'][0].index(_e2)\n",
    "            EntTokensdata=EntTokens(output, reverse, e11, e22)\n",
    "            y_pred=neigh.predict(EntTokensdata.reshape(1,-1))\n",
    "            if y_pred[0]==0:\n",
    "                print(\"no relations predicted for head entity \",datast['head'],\" and child entity \",datast['child'])\n",
    "            else: \n",
    "                l=int(y_pred[0])\n",
    "                print(\"predicted (\",datast['head'][0],\",\",label[l],\",\",datast['child'])\n",
    "                r={}\n",
    "                r['relation']=label[l]\n",
    "                r['head']=datast['head'][0]\n",
    "                r['child']=datast['child']\n",
    "                r['text']=i[0]['text']\n",
    "                rel.append(r)\n",
    "    return rel\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:00<00:00, 836.35ex/s]\n",
      "The following columns in the test set  don't have a corresponding argument in `DistilBertForTokenClassification.forward` and have been ignored: text.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 3\n",
      "  Batch size = 8\n",
      "63it [1:10:47, 92.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['instagram was bought by facebook to compete with tiktok.', ['tiktok', 'instagram', 'facebook']], ['eurecom is based in france and offers educational courses.', ['eurecom', 'france']], ['eurecom will partner with inria for various research projects.', ['eurecom', 'inria']]]\n",
      "predicting sentence : ['instagram was bought by facebook to compete with tiktok.']\n",
      "predicting on  6  different combinations of entities\n",
      "predicted ( instagram , PARTNERSHIP , ['tiktok']\n",
      "predicted ( facebook , PARTNERSHIP , ['tiktok']\n",
      "no relations predicted for head entity  ['instagram']  and child entity  ['tiktok']\n",
      "predicted ( instagram , PARTNERSHIP , ['facebook']\n",
      "predicted ( facebook , PARTNERSHIP , ['tiktok']\n",
      "predicted ( instagram , PARTNERSHIP , ['facebook']\n",
      "predicting sentence : ['eurecom is based in france and offers educational courses.']\n",
      "predicting on  2  different combinations of entities\n",
      "no relations predicted for head entity  ['eurecom']  and child entity  ['france']\n",
      "predicted ( eurecom , BASED_IN , ['france']\n",
      "predicting sentence : ['eurecom will partner with inria for various research projects.']\n",
      "predicting on  2  different combinations of entities\n",
      "predicted ( eurecom , PARTNERSHIP , ['inria']\n",
      "predicted ( eurecom , PARTNERSHIP , ['inria']\n"
     ]
    }
   ],
   "source": [
    "inputs=\"Instagram was bought by Facebook to compete with Tiktok. Eurecom is based in France and offers educational courses. Eurecom will partner with INRIA for various research projects.\"\n",
    "rel=make_pred(inputs,labels)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "condakaggle",
   "language": "python",
   "name": "condakaggle"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}