coreference classifier testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from utils.data_helper import get_markable_dataframe, get_embedding_variables, get_sentence_variables, get_document_id_variables\n",
    "from model_builders.coreference_classifier import CoreferenceClassifierModelBuilder\n",
    "from functools import reduce\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import load_model\n",
    "from utils.clusterers import BestFirstClusterer, get_anaphora_scores_by_antecedent, ClosestFirstClusterer\n",
    "from utils.per_document_scorers import PerDocumentScorer\n",
    "from utils.scorers import MUCScorer, B3Scorer, CEAFeScorer, AverageScorer\n",
    "from utils.data_structures import UFDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_id_by_markable_id, markable_ids_by_sentence_id = get_sentence_variables('data/full.xml')\n",
    "document_id_by_sentence_id, document_id_by_markable_id, sentence_ids_by_document_id, markable_ids_by_document_id = get_document_id_variables('data/document_id.csv', markable_ids_by_sentence_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_indexes_file_path = 'helper_files/embedding/embedding_indexes.txt'\n",
    "indexed_embedding_file_path = 'helper_files/embedding/indexed_embedding.txt'\n",
    "\n",
    "word_vector, embedding_matrix, idx_by_word, word_by_idx = get_embedding_variables(embedding_indexes_file_path, indexed_embedding_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>is_pronoun</th>\n",
       "      <th>entity_type</th>\n",
       "      <th>is_proper_name</th>\n",
       "      <th>is_first_person</th>\n",
       "      <th>previous_words</th>\n",
       "      <th>next_words</th>\n",
       "      <th>is_singleton</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1916</td>\n",
       "      <td>[1263, 1264, 1968, 1395]</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 1, 0, 0, 0]</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[999, 379, 1161, 213, 27, 1263, 1969, 1188, 14...</td>\n",
       "      <td>[0.0, 1.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1917</td>\n",
       "      <td>[213]</td>\n",
       "      <td>1</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 1, 0, 0, 0]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[1263, 1264, 1968, 1395, 999, 379, 1161]</td>\n",
       "      <td>[27, 1263, 1969, 1188, 1470, 25, 1161, 63, 424...</td>\n",
       "      <td>[0.0, 1.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1918</td>\n",
       "      <td>[1263, 1969, 1188]</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 1, 0, 1, 0, 0, 0, 0, 0]</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[1263, 1264, 1968, 1395, 999, 379, 1161, 213, 27]</td>\n",
       "      <td>[1470, 25, 1161, 63, 424, 1223, 25, 1415, 1161...</td>\n",
       "      <td>[0.0, 1.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1919</td>\n",
       "      <td>[1470, 25, 1161]</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 1, 0, 0, 0]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[1968, 1395, 999, 379, 1161, 213, 27, 1263, 19...</td>\n",
       "      <td>[63, 424, 1223, 25, 1415, 1161, 876, 344, 213,...</td>\n",
       "      <td>[0.0, 1.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1920</td>\n",
       "      <td>[424]</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[1161, 213, 27, 1263, 1969, 1188, 1470, 25, 11...</td>\n",
       "      <td>[1223, 25, 1415, 1161, 876, 344, 213, 406, 122...</td>\n",
       "      <td>[0.0, 1.0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id                      text  is_pronoun                     entity_type  \\\n",
       "0  1916  [1263, 1264, 1968, 1395]           0  [0, 0, 0, 0, 1, 0, 1, 0, 0, 0]   \n",
       "1  1917                     [213]           1  [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]   \n",
       "2  1918        [1263, 1969, 1188]           0  [0, 0, 1, 0, 1, 0, 0, 0, 0, 0]   \n",
       "3  1919          [1470, 25, 1161]           0  [0, 0, 0, 0, 0, 1, 1, 0, 0, 0]   \n",
       "4  1920                     [424]           0  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0]   \n",
       "\n",
       "   is_proper_name  is_first_person  \\\n",
       "0               1                0   \n",
       "1               0                0   \n",
       "2               1                0   \n",
       "3               0                0   \n",
       "4               0                0   \n",
       "\n",
       "                                      previous_words  \\\n",
       "0                                                 []   \n",
       "1           [1263, 1264, 1968, 1395, 999, 379, 1161]   \n",
       "2  [1263, 1264, 1968, 1395, 999, 379, 1161, 213, 27]   \n",
       "3  [1968, 1395, 999, 379, 1161, 213, 27, 1263, 19...   \n",
       "4  [1161, 213, 27, 1263, 1969, 1188, 1470, 25, 11...   \n",
       "\n",
       "                                          next_words is_singleton  \n",
       "0  [999, 379, 1161, 213, 27, 1263, 1969, 1188, 14...   [0.0, 1.0]  \n",
       "1  [27, 1263, 1969, 1188, 1470, 25, 1161, 63, 424...   [0.0, 1.0]  \n",
       "2  [1470, 25, 1161, 63, 424, 1223, 25, 1415, 1161...   [0.0, 1.0]  \n",
       "3  [63, 424, 1223, 25, 1415, 1161, 876, 344, 213,...   [0.0, 1.0]  \n",
       "4  [1223, 25, 1415, 1161, 876, 344, 213, 406, 122...   [0.0, 1.0]  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "markables = get_markable_dataframe(\"data/testing/markables_with_predicted_singleton.csv\", word_vector, idx_by_word)\n",
    "label_markables = get_markable_dataframe(\"data/testing/markables.csv\", word_vector, idx_by_word)\n",
    "singletons = set(markables[markables['is_singleton'].map(lambda x: True if x[1] > 0 else False)]['id'])\n",
    "label_singletons = set(label_markables[label_markables['is_singleton'].map(lambda x: True if x[1] > 0 else False)]['id'])\n",
    "markables.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>m1_id</th>\n",
       "      <th>m2_id</th>\n",
       "      <th>is_exact_match</th>\n",
       "      <th>is_words_match</th>\n",
       "      <th>is_substring</th>\n",
       "      <th>is_abbreviation</th>\n",
       "      <th>is_appositive</th>\n",
       "      <th>is_nearest_candidate</th>\n",
       "      <th>sentence_distance</th>\n",
       "      <th>word_distance</th>\n",
       "      <th>markable_distance</th>\n",
       "      <th>is_coreference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1916</td>\n",
       "      <td>1917</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1916</td>\n",
       "      <td>1918</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1916</td>\n",
       "      <td>1919</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1916</td>\n",
       "      <td>1920</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1916</td>\n",
       "      <td>1921</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   m1_id  m2_id  is_exact_match  is_words_match  is_substring  \\\n",
       "0   1916   1917               0               0             0   \n",
       "1   1916   1918               0               0             0   \n",
       "2   1916   1919               0               0             0   \n",
       "3   1916   1920               0               0             0   \n",
       "4   1916   1921               0               0             0   \n",
       "\n",
       "   is_abbreviation  is_appositive  is_nearest_candidate  sentence_distance  \\\n",
       "0                0              0                     1                  0   \n",
       "1                0              0                     0                  0   \n",
       "2                0              0                     0                  0   \n",
       "3                0              0                     0                  0   \n",
       "4                0              0                     0                  0   \n",
       "\n",
       "   word_distance  markable_distance  is_coreference  \n",
       "0              3                  1               1  \n",
       "1              5                  2               0  \n",
       "2              8                  3               0  \n",
       "3             12                  4               0  \n",
       "4             13                  5               0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs = pd.read_csv(\"data/testing/mention_pairs.csv\")\n",
    "\n",
    "label = np.vstack(to_categorical(pairs.is_coreference, num_classes=2))\n",
    "label_chains = ClosestFirstClusterer().get_chains(get_anaphora_scores_by_antecedent(pairs.m1_id, pairs.m2_id, label))\n",
    "\n",
    "pairs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_text_length = 10\n",
    "max_prev_words_length = 10\n",
    "max_next_words_length = 10\n",
    "\n",
    "def get_data(markable_ids):\n",
    "    indices = reduce(lambda a, b: a + [b], map(lambda a: markables.index[markables['id'] == a].tolist()[0], markable_ids), [])\n",
    "    data = markables.loc[indices]\n",
    "    \n",
    "    data_text = pad_sequences(data.text, maxlen=max_text_length, padding='post')\n",
    "    data_previous_words = pad_sequences(data.previous_words.map(lambda seq: seq[(-1*max_prev_words_length):]), maxlen=max_prev_words_length, padding='pre')\n",
    "    data_next_words = pad_sequences(data.next_words.map(lambda seq: seq[:max_next_words_length]), maxlen=max_next_words_length, padding='post')\n",
    "    data_syntactic = data[['is_pronoun', 'entity_type', 'is_proper_name', 'is_first_person']]\n",
    "\n",
    "    data_syntactic = np.array(list(map(lambda p: reduce(lambda x,y: x + y, [i if type(i) is list else [i] for i in p]), data_syntactic.values)))\n",
    "    is_singleton = np.vstack(data.is_singleton)\n",
    "    \n",
    "    return data_text, data_previous_words, data_next_words, data_syntactic, is_singleton\n",
    "\n",
    "def get_pair_data(markable_ids_1, markable_ids_2):\n",
    "    text_1, prev_1, next_1, syntactic_1, is_singleton_1 = get_data(markable_ids_1)\n",
    "    text_2, prev_2, next_2, syntactic_2, is_singleton_2 = get_data(markable_ids_2)\n",
    "    \n",
    "    return text_1, text_2, prev_1, prev_2, next_1, next_2, syntactic_1, syntactic_2, is_singleton_1, is_singleton_2\n",
    "\n",
    "def get_relation_data(mention_pairs):\n",
    "    return mention_pairs[['is_exact_match', 'is_words_match', 'is_substring', 'is_abbreviation', 'is_appositive', 'is_nearest_candidate', 'sentence_distance', 'word_distance', 'markable_distance']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute Baseline Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MUC:  (0.6395348837209303, 0.7051282051282052, 0.6707317073170733)\n",
      "B3:  (0.5041087231352718, 0.6379818594104308, 0.5631991462778547)\n",
      "CEAFe:  (0.572617259283926, 0.572617259283926, 0.572617259283926)\n",
      "Average:  (0.6021827042929514, 0.6021827042929514, 0.6021827042929514)\n"
     ]
    }
   ],
   "source": [
    "baseline_result_file_path = 'baseline/suherik_and_purwarianti/test_result.txt'\n",
    "\n",
    "baseline_ufds = UFDS()\n",
    "\n",
    "for m1, m2 in zip(pairs.m1_id, pairs.m2_id):\n",
    "    baseline_ufds.init_id(m1, m2)\n",
    "    \n",
    "for line in open(baseline_result_file_path, 'r').readlines():\n",
    "    line = line.split(', ')\n",
    "    m1_id, m2_id = int(line[0]), int(line[1])\n",
    "    \n",
    "    if document_id_by_markable_id[m1_id] == document_id_by_markable_id[m2_id]:\n",
    "        baseline_ufds.join(m1_id, m2_id)\n",
    "\n",
    "baseline_chains = baseline_ufds.get_chain_list()\n",
    "\n",
    "print('MUC: ', MUCScorer().get_scores(baseline_chains, label_chains))\n",
    "print('B3: ', B3Scorer().get_scores(baseline_chains, label_chains))\n",
    "print('CEAFe: ', PerDocumentScorer(document_id_by_markable_id, 'ceafe').get_scores(baseline_chains, label_chains))\n",
    "print('Average: ', AverageScorer([MUCScorer(), B3Scorer(), PerDocumentScorer(document_id_by_markable_id, 'ceafe')]).get_scores(baseline_chains, label_chains))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_1, text_2, prev_1, prev_2, next_1, next_2, syntactic_1, syntactic_2, is_singleton_1, is_singleton_2 = get_pair_data(pairs.m1_id, pairs.m2_id)\n",
    "relation = get_relation_data(pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {}\n",
    "\n",
    "def get_model(hyperparameter, features, data_generation, epoch):\n",
    "    name = hyperparameter + '_hyperparameter/' + '_'.join([*features, data_generation, str(epoch)])\n",
    "    \n",
    "    if name not in models:\n",
    "        models[name] = load_model(f'models/coreference_classifiers/{name}.model')\n",
    "    \n",
    "    return models[name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_thresholds = [0.1, 0.01, 0.001, 0.0001, 0.00001]\n",
    "thresholds = [0] + [base * multiplier for base in base_thresholds for multiplier in range(1, 10)]\n",
    "\n",
    "muc_scorer = MUCScorer()\n",
    "b3_scorer = B3Scorer()\n",
    "ceafe_scorer = CEAFeScorer()\n",
    "average_scorer = AverageScorer([muc_scorer, b3_scorer])\n",
    "\n",
    "def get_sorted_scores(clusterer, pred, thresholds=thresholds):\n",
    "    scores = [] # will be a tuple (average_f1, (prec_muc, rec_muc, f1_muc), (prec_b3, rec_b3, f1_b3), threshold)\n",
    "    \n",
    "    for threshold in thresholds:\n",
    "        predicted_chains = clusterer.get_chains(pred, threshold)\n",
    "        \n",
    "#         avg_f1 = average_scorer.get_scores(predicted_chains, label_chains)[2]\n",
    "        muc = muc_scorer.get_scores(predicted_chains, label_chains)\n",
    "        b3 = b3_scorer.get_scores(predicted_chains, label_chains)\n",
    "#         ceafe = ceafe_scorer.get_scores(predicted_chains, label_chains)\n",
    "        avg_f1 = (muc[2] + b3[2] ) / 2\n",
    "        \n",
    "        scores.append((avg_f1, muc, b3, threshold))\n",
    "    \n",
    "    return sorted(scores, reverse=True)\n",
    "\n",
    "def reorder_score(score):\n",
    "    avg_f1, muc, b3, threshold = score\n",
    "    return muc, b3, avg_f1, threshold \n",
    "\n",
    "def evaluate(hyperparameter, features, data_generation, epoch):\n",
    "    model = get_model(hyperparameter, features, data_generation, epoch)\n",
    "    \n",
    "    test_features = []\n",
    "    if 'words' in features:\n",
    "        test_features.extend([text_1, text_2])\n",
    "    if 'context' in features:\n",
    "        test_features.extend([prev_1, prev_2, next_1, next_2])\n",
    "    if 'syntactic' in features:\n",
    "        test_features.extend([syntactic_1, syntactic_2, relation])\n",
    "    \n",
    "    print('getting anaphora scores by antecedent dict')\n",
    "    raw_pred = model.predict(test_features, verbose=1)\n",
    "    pred_without_singleton_classifier = get_anaphora_scores_by_antecedent(pairs.m1_id, pairs.m2_id, raw_pred)\n",
    "    pred_with_singleton_classifier = get_anaphora_scores_by_antecedent(pairs.m1_id, pairs.m2_id, raw_pred, singletons)\n",
    "    pred_with_label_singletons = get_anaphora_scores_by_antecedent(pairs.m1_id, pairs.m2_id, raw_pred, label_singletons)\n",
    "        \n",
    "    print('get sorted_scores_with_sc_best')\n",
    "    sorted_scores_with_sc_best = get_sorted_scores(BestFirstClusterer(), pred_with_singleton_classifier)\n",
    "    print('With singleton classifier, best-first:', ' '.join(str(x) if type(x) != tuple else ' '.join(str(y) for y in x) for x in reorder_score(sorted_scores_with_sc_best[0])))\n",
    "    \n",
    "    print('get sorted_scores_with_label_singletons_best')\n",
    "    sorted_scores_with_label_singletons_best = get_sorted_scores(BestFirstClusterer(), pred_with_label_singletons, [sorted_scores_with_sc_best[0][3]])\n",
    "    print('With label singletons, best-first:', ' '.join(str(x) if type(x) != tuple else ' '.join(str(y) for y in x) for x in reorder_score(sorted_scores_with_label_singletons_best[0])))\n",
    "    \n",
    "    print('get sorted_scores_without_sc_best')\n",
    "    sorted_scores_without_sc_best = get_sorted_scores(BestFirstClusterer(), pred_without_singleton_classifier, [sorted_scores_with_sc_best[0][3]])\n",
    "    print('Without singleton classifier, best-first:', ' '.join(str(x) if type(x) != tuple else ' '.join(str(y) for y in x) for x in reorder_score(sorted_scores_without_sc_best[0])))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Budi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Words + Context + Syntactic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting anaphora scores by antecedent dict\n",
      "24703/24703==============================] - 4s 152us/sample\n",
      "get sorted_scores_with_sc_best\n",
      "With singleton classifier, best-first: 0.5974025974025974 0.5897435897435898 0.5935483870967742 0.3984814635913538 0.5231065759637188 0.45236757652566695 0.5229579818112206 0.07\n",
      "get sorted_scores_with_label_singletons_best\n",
      "With label singletons, best-first: 0.8441558441558441 0.8333333333333334 0.8387096774193548 0.6004799548277809 0.7988208616780045 0.6855937040524324 0.7621516907358936 0.07\n",
      "get sorted_scores_without_sc_best\n",
      "Without singleton classifier, best-first: 0.2831858407079646 0.8205128205128205 0.42105263157894735 0.10169835920357859 0.7497732426303857 0.17910335091862667 0.300077991248787 0.07\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate('wu_ma', ['words', 'context', 'syntactic'], 'budi', 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting anaphora scores by antecedent dict\n",
      "24703/24703==============================] - 4s 156us/sample\n",
      "get sorted_scores_with_sc_best\n",
      "With singleton classifier, best-first: 0.6052631578947368 0.5897435897435898 0.5974025974025974 0.41207570207570215 0.530249433106576 0.463752688461121 0.5305776429318592 0.02\n",
      "get sorted_scores_with_label_singletons_best\n",
      "With label singletons, best-first: 0.85 0.8717948717948718 0.8607594936708861 0.6002946760841499 0.8585714285714287 0.7065704741026533 0.7836649838867698 0.02\n",
      "get sorted_scores_without_sc_best\n",
      "Without singleton classifier, best-first: 0.14316239316239315 0.8589743589743589 0.2454212454212454 0.031411545720768944 0.8353968253968254 0.060546497819591656 0.15298387162041854 0.02\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate('wu_ma', ['words', 'context', 'syntactic'], 'budi', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting anaphora scores by antecedent dict\n",
      "24703/24703==============================] - 4s 154us/sample\n",
      "get sorted_scores_with_sc_best\n",
      "With singleton classifier, best-first: 0.6133333333333333 0.5897435897435898 0.6013071895424836 0.4256945301889123 0.530249433106576 0.4722542156782639 0.5367807026103738 0.06\n",
      "get sorted_scores_with_label_singletons_best\n",
      "With label singletons, best-first: 0.8701298701298701 0.8589743589743589 0.8645161290322582 0.6213088498802783 0.832154195011338 0.7114384746728767 0.7879773018525674 0.06\n",
      "get sorted_scores_without_sc_best\n",
      "Without singleton classifier, best-first: 0.2882882882882883 0.8205128205128205 0.4266666666666667 0.10332250309030494 0.797233560090703 0.1829362331651084 0.30480144991588753 0.06\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate('wu_ma', ['words', 'context', 'syntactic'], 'budi', 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gilang"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Words + Context + Syntactic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting anaphora scores by antecedent dict\n",
      "24703/24703==============================] - 5s 211us/sample\n",
      "get sorted_scores_with_sc_best\n",
      "With singleton classifier, best-first: 0.6388888888888888 0.5897435897435898 0.6133333333333332 0.45876250186595013 0.530249433106576 0.4919223882808494 0.5526278608070913 0.009000000000000001\n",
      "get sorted_scores_with_label_singletons_best\n",
      "With label singletons, best-first: 0.8648648648648649 0.8205128205128205 0.8421052631578947 0.6282041191132101 0.7886167800453515 0.6993294775938265 0.7707173703758605 0.009000000000000001\n",
      "get sorted_scores_without_sc_best\n",
      "Without singleton classifier, best-first: 0.32275132275132273 0.782051282051282 0.4569288389513108 0.13433062509149465 0.7337414965986393 0.2270870159999155 0.34200792747561315 0.009000000000000001\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate('wu_ma', ['words', 'context', 'syntactic'], 'gilang', 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting anaphora scores by antecedent dict\n",
      "24703/24703==============================] - 6s 224us/sample\n",
      "get sorted_scores_with_sc_best\n",
      "With singleton classifier, best-first: 0.6153846153846154 0.6153846153846154 0.6153846153846154 0.4240414476827521 0.5569160997732426 0.48147957023853094 0.5484320928115731 3.0000000000000004e-05\n",
      "get sorted_scores_with_label_singletons_best\n",
      "With label singletons, best-first: 0.8607594936708861 0.8717948717948718 0.8662420382165605 0.605438408664215 0.8421541950113378 0.7044419740511945 0.7853420061338775 3.0000000000000004e-05\n",
      "get sorted_scores_without_sc_best\n",
      "Without singleton classifier, best-first: 0.2537878787878788 0.8589743589743589 0.391812865497076 0.08201345656792812 0.8150113378684809 0.14903021048072965 0.27042153798890284 3.0000000000000004e-05\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate('wu_ma', ['words', 'context', 'syntactic'], 'gilang', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting anaphora scores by antecedent dict\n",
      "24703/24703==============================] - 8s 315us/sample\n",
      "get sorted_scores_with_sc_best\n",
      "With singleton classifier, best-first: 0.6301369863013698 0.5897435897435898 0.609271523178808 0.45828469283525464 0.5169160997732426 0.48583786601018847 0.5475546945944982 0.04\n",
      "get sorted_scores_with_label_singletons_best\n",
      "With label singletons, best-first: 0.8493150684931506 0.7948717948717948 0.8211920529801324 0.6436062065275548 0.7522675736961452 0.6937075346779171 0.7574497938290248 0.04\n",
      "get sorted_scores_without_sc_best\n",
      "Without singleton classifier, best-first: 0.3905325443786982 0.8461538461538461 0.5344129554655871 0.17562541076535385 0.8108390022675737 0.2887158035432525 0.4115643795044198 0.04\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate('wu_ma', ['words', 'context', 'syntactic'], 'gilang', 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Soon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Words + Context + Syntactic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting anaphora scores by antecedent dict\n",
      "24703/24703==============================] - 6s 228us/sample\n",
      "get sorted_scores_with_sc_best\n",
      "With singleton classifier, best-first: 0.618421052631579 0.6025641025641025 0.6103896103896105 0.4337391774891775 0.5473922902494331 0.48398301255986265 0.5471863114747366 0.2\n",
      "get sorted_scores_with_label_singletons_best\n",
      "With label singletons, best-first: 0.8433734939759037 0.8974358974358975 0.8695652173913043 0.6004121162284428 0.8847619047619049 0.7153663612324334 0.7924657893118688 0.2\n",
      "get sorted_scores_without_sc_best\n",
      "Without singleton classifier, best-first: 0.24056603773584906 0.6538461538461539 0.3517241379310345 0.14909822903520387 0.5973696145124716 0.23863520009100944 0.295179669011022 0.2\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate('wu_ma', ['words', 'context', 'syntactic'], 'soon', 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting anaphora scores by antecedent dict\n",
      "24703/24703==============================] - 6s 224us/sample\n",
      "get sorted_scores_with_sc_best\n",
      "With singleton classifier, best-first: 0.6527777777777778 0.6025641025641025 0.6266666666666667 0.4990502942750134 0.5426303854875283 0.5199287244567992 0.5732976955617329 0.2\n",
      "get sorted_scores_with_label_singletons_best\n",
      "With label singletons, best-first: 0.8625 0.8846153846153846 0.8734177215189874 0.6445744404507292 0.8575510204081633 0.7359644497623874 0.8046910856406875 0.2\n",
      "get sorted_scores_without_sc_best\n",
      "Without singleton classifier, best-first: 0.1408839779005525 0.6538461538461539 0.2318181818181818 0.07901802438306162 0.62437641723356 0.14028257274191186 0.18605037728004684 0.2\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate('wu_ma', ['words', 'context', 'syntactic'], 'soon', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting anaphora scores by antecedent dict\n",
      "24703/24703==============================] - 5s 222us/sample\n",
      "get sorted_scores_with_sc_best\n",
      "With singleton classifier, best-first: 0.7636363636363637 0.5384615384615384 0.6315789473684211 0.6763046314416177 0.4665759637188208 0.5521967675688427 0.5918878574686319 0.8\n",
      "get sorted_scores_with_label_singletons_best\n",
      "With label singletons, best-first: 0.864406779661017 0.6538461538461539 0.7445255474452555 0.7715538847117793 0.5909070294784582 0.6692545956352384 0.7068900715402469 0.8\n",
      "get sorted_scores_without_sc_best\n",
      "Without singleton classifier, best-first: 0.36231884057971014 0.6410256410256411 0.46296296296296297 0.25697879248411165 0.5986167800453516 0.3595900264934673 0.41127649472821515 0.8\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate('wu_ma', ['words', 'context', 'syntactic'], 'soon', 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_markable_text(idx):\n",
    "    return [word_by_idx[x] for x in markables[markables['id'] == idx].text.values[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0516 13:05:31.470775 139621331662656 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:111: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0516 13:05:31.476242 139621331662656 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:135: calling RandomNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0516 13:05:31.477232 139621331662656 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:96: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0516 13:05:31.502385 139621331662656 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:96: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0516 13:05:31.503385 139621331662656 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1253: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24703/24703==============================] - 5s 203us/sample\n"
     ]
    }
   ],
   "source": [
    "best_model = get_model('proposed', ['words', 'context', 'syntactic'], 'budi', 20)\n",
    "raw_pred_best = best_model.predict([text_1, text_2, prev_1, prev_2, next_1, next_2, syntactic_1, syntactic_2, relation], verbose=1)\n",
    "pred_best_wo_sc = get_anaphora_scores_by_antecedent(pairs.m1_id, pairs.m2_id, raw_pred_best)\n",
    "pred_best_w_sc = get_anaphora_scores_by_antecedent(pairs.m1_id, pairs.m2_id, raw_pred_best, singletons)\n",
    "pred_best_w_label_sc = get_anaphora_scores_by_antecedent(pairs.m1_id, pairs.m2_id, raw_pred_best, label_singletons)\n",
    "pred_chains_best_wo_sc = BestFirstClusterer().get_chains(pred_best_wo_sc, 0.2)\n",
    "pred_chains_best_w_sc = BestFirstClusterer().get_chains(pred_best_w_sc, 0.2)\n",
    "pred_chains_best_w_label_sc = BestFirstClusterer().get_chains(pred_best_w_label_sc, 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_chains(chains):\n",
    "    return [[(id, ' '.join(get_markable_text(id))) for id in chain] for chain in chains if len(chain) > 1]\n",
    "\n",
    "def save_chains(chains, file_path):\n",
    "    chains = convert_chains(chains)\n",
    "    \n",
    "    with open(file_path, 'w') as f:\n",
    "        f.write(json.dumps(chains, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_chains(pred_chains_best_w_sc, 'result/with_singleton_classifier.json')\n",
    "save_chains(pred_chains_best_wo_sc, 'result/without_singleton_classifier.json')\n",
    "save_chains(pred_chains_best_w_label_sc, 'result/with_label_singleton_classifier.json')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python TA",
   "language": "python",
   "name": "ta-v2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}