M2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')\n",
    "\n",
    "# Eventually, for Anaconda warnings.\n",
    "# Can be commented out.\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load basic libraries\n",
    "import seaborn; seaborn.set()\n",
    "import pickle, copy, json\n",
    "import numpy as np\n",
    "import scipy.stats\n",
    "from sklearn.metrics import make_scorer, confusion_matrix\n",
    "from sklearn.model_selection import cross_val_score, RandomizedSearchCV, train_test_split\n",
    "from sklearn.externals import joblib\n",
    "from sklearn_crfsuite import scorers, metrics\n",
    "import sklearn_crfsuite\n",
    "from multiprocessing import Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# rememebr to save this dataset from before!\n",
    "data = pickle.load(open(\"dataset/data.p\", \"rb\"))\n",
    "print(len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Generic Tagged BE Tag consolidation\n",
    "correspondances = {\n",
    "     'b-primary-full': 'b-primary', \n",
    "     'i-primary-full': 'i-primary', \n",
    "     'e-primary-full': 'e-primary', \n",
    "     'b-primary-partial': 'b-primary', \n",
    "     'i-primary-partial': 'i-primary', \n",
    "     'e-primary-partial': 'e-primary', \n",
    "     'b-meta-annotation': 'b-meta-annotation', \n",
    "     'i-meta-annotation': 'i-meta-annotation', \n",
    "     'e-meta-annotation': 'e-meta-annotation', \n",
    "     'b-secondary-full': 'b-secondary', \n",
    "     'i-secondary-full': 'i-secondary', \n",
    "     'e-secondary-full': 'e-secondary', \n",
    "     'b-secondary-partial': 'b-secondary', \n",
    "     'i-secondary-partial': 'i-secondary', \n",
    "     'e-secondary-partial': 'e-secondary', \n",
    "     'o': 'o', \n",
    "}\n",
    "# define supporting functions\n",
    "window = 2\n",
    "from code.feature_extraction_words import word2features, generate_featuresLight\n",
    "def text2features(text):\n",
    "    return [word2features(text, i, window = window) for i in range(len(text))]\n",
    "def text2featuresL(text):\n",
    "    return [word2features(text, i, window = window, feature_function=generate_featuresLight) for i in range(len(text))]\n",
    "# With extra Specifc Tags. Adding specific tags improves performances\n",
    "def text2featuresEX(text, extra_labels):\n",
    "    return [word2features(text, i, extra_labels, window = window) for i in range(len(text))]\n",
    "def text2featuresLEX(text, extra_labels):\n",
    "    return [word2features(text, i, extra_labels, window = window, feature_function=generate_featuresLight) for i in range(len(text))]\n",
    "\n",
    "# create generic tags Y\n",
    "def text2labelsG(text):\n",
    "    return [correspondances[token[2][0]] for token in text]\n",
    "\n",
    "# create beginend tags Y\n",
    "def text2labelsBE(text):\n",
    "    return [token[2][2] for token in text]\n",
    "\n",
    "# create tagged-beginend tags Y\n",
    "def text2labelsTBE(text):\n",
    "    return [correspondances[token[2][3]] for token in text]\n",
    "\n",
    "# create specific tags Y\n",
    "def text2labelsS(text):\n",
    "    return [correspondances[token[2][1]] for token in text]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare data for CRF\n",
    "annotated_data = list()\n",
    "annotated_labels = list()\n",
    "for doc in data:\n",
    "    ar_data_ann = list()\n",
    "    ar_labels_ann = list()\n",
    "    for page in doc[\"pages\"].values():\n",
    "        if page[\"is_annotated\"]:\n",
    "            ar_data_ann.extend(page[\"offsets\"])\n",
    "            ar_labels_ann.extend(page[\"specific_tags\"])\n",
    "    if len(ar_data_ann) > 0:\n",
    "        annotated_data.append(ar_data_ann)\n",
    "        annotated_labels.append(ar_labels_ann)\n",
    "print(len(annotated_data))\n",
    "print(len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define train and test sets for experiments\n",
    "%%time\n",
    "d = [text2featuresEX(text, lab) for text, lab in zip(annotated_data, annotated_labels)]\n",
    "l = [text2labelsTBE(text) for text in annotated_data]\n",
    "# Clean tag space\n",
    "labels_to_keep = sorted(list(set([x for y in l for x in y])))\n",
    "# VALIDATION set\n",
    "X_rest, X_valid, y_rest, y_valid = train_test_split(d, l, test_size=0.1)\n",
    "# TRAIN/TEST\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_rest, y_rest, test_size=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Count labels\n",
    "counts = {x:0 for x in labels_to_keep}\n",
    "for c in counts.keys():\n",
    "    counts[c] = len([x for y in l for x in y if x==c])\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# An example use of CRFs\n",
    "%%time\n",
    "crf = sklearn_crfsuite.CRF( \n",
    "    algorithm='lbfgs',\n",
    "    c1=0.1,\n",
    "    c2=0.1,\n",
    "    max_iterations=100,\n",
    "    all_possible_transitions=False\n",
    ")\n",
    "crf.fit(X_train, y_train)\n",
    "y_pred = crf.predict(X_test)\n",
    "print(metrics.flat_classification_report(\n",
    "    y_test, y_pred, labels=labels_to_keep, digits=3\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters search\n",
    "%%time\n",
    "crf = sklearn_crfsuite.CRF( \n",
    "    max_iterations=100,\n",
    "    algorithm = 'lbfgs',\n",
    "    all_possible_transitions=False\n",
    ")\n",
    "params_space = {\n",
    "    'c1': scipy.stats.expon(scale=0.5),\n",
    "    'c2': scipy.stats.expon(scale=0.05)\n",
    "}\n",
    "\n",
    "scorer = make_scorer(metrics.flat_f1_score, \n",
    "                        average='weighted', labels=labels_to_keep)\n",
    "\n",
    "# search\n",
    "rs = RandomizedSearchCV(crf, params_space, \n",
    "                        cv=3, \n",
    "                        verbose=1, \n",
    "                        n_jobs=-15, \n",
    "                        n_iter=5, \n",
    "                        scoring=scorer)\n",
    "rs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print('best params:', rs.best_params_)\n",
    "print('best CV score:', rs.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# classification report\n",
    "crf = rs.best_estimator_\n",
    "y_pred = crf.predict(X_test)\n",
    "print(metrics.flat_classification_report(\n",
    "    y_test, y_pred, labels=labels_to_keep, digits=3\n",
    "))\n",
    "# Confusion matrices\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from code.support_functions import flatten_predictions\n",
    "\n",
    "print(confusion_matrix(flatten_predictions(y_test), flatten_predictions(y_pred), labels=labels_to_keep))\n",
    "plt.imshow(np.log(confusion_matrix(flatten_predictions(y_test), flatten_predictions(y_pred), labels=labels_to_keep)),\n",
    "           cmap='Blues', interpolation='nearest')\n",
    "plt.grid(False)\n",
    "plt.ylabel('Ground truth', fontsize=16)\n",
    "plt.xlabel('Predicted', fontsize=16)\n",
    "plt.xticks(np.arange(0, len(labels_to_keep), 1))\n",
    "plt.yticks(np.arange(0, len(labels_to_keep), 1))\n",
    "plt.title(\"Confusion Matrix Model 2\", fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# K-fold validation\n",
    "scorer = make_scorer(metrics.flat_f1_score, \n",
    "                        average='weighted', labels=labels_to_keep)\n",
    "# OR rs.best_params_\n",
    "crf = sklearn_crfsuite.CRF(\n",
    "    algorithm='lbfgs',\n",
    "    c2= 0.093645710804034776, c1= 0.44740028179508301,\n",
    "    max_iterations=200, \n",
    "    all_possible_transitions=True\n",
    ")\n",
    "k = 5\n",
    "\n",
    "cv = cross_val_score(crf, X_rest, y_rest, cv=k, scoring=scorer, n_jobs=-2)\n",
    "print(\"%d-fold validation mean: \"%k,cv.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Learning curves\n",
    "from code.support_functions import plot_learning_curve\n",
    "\n",
    "# Slices of data for learning curves\n",
    "train_sizes=np.linspace(0.1, 1.0, 10)\n",
    "title = \"Learning Curves for Model 2\"\n",
    "message = \"M2\"\n",
    "# Cross validation scheme with 80-20 splits and 5 iterations per train data size (to evaluate variance)\n",
    "cv = model_selection.ShuffleSplit(test_size=0.2, random_state=0)\n",
    "estimator = sklearn_crfsuite.CRF(\n",
    "    algorithm='lbfgs',\n",
    "    c2= 0.093645710804034776, c1= 0.44740028179508301,\n",
    "    max_iterations=200, \n",
    "    all_possible_transitions=True\n",
    ")\n",
    "plot_learning_curve(estimator, title, X_rest, y_rest, labels_to_keep, cv=cv, train_sizes=train_sizes, n_jobs=-2, message=message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# VALIDATION\n",
    "%%time\n",
    "\n",
    "crf = sklearn_crfsuite.CRF( \n",
    "    algorithm='lbfgs',\n",
    "    c2= 0.093645710804034776, c1= 0.44740028179508301,\n",
    "    max_iterations=500,\n",
    "    all_possible_transitions=True\n",
    ")\n",
    "crf.fit(X_rest, y_rest)\n",
    "y_pred = crf.predict(X_valid)\n",
    "print(metrics.flat_classification_report(\n",
    "    y_valid, y_pred, labels=labels_to_keep, digits=3\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Train final models for task 1\n",
    "crf = sklearn_crfsuite.CRF(\n",
    "    algorithm='lbfgs',\n",
    "    c2= 0.093645710804034776, c1= 0.44740028179508301,\n",
    "    max_iterations=500, \n",
    "    all_possible_transitions=True\n",
    ")\n",
    "crf.fit(d, l)\n",
    "\n",
    "# save model\n",
    "#joblib.dump(crf,'models/modelM2_ALL_L.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load model\n",
    "crf1 = joblib.load('models/modelM2_ALL_L.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_document(doc):\n",
    "    for page in doc[\"pages\"].values():\n",
    "        if not page[\"is_annotated\"]:\n",
    "            data_to_tag = [text2featuresEX(page[\"offsets\"],page[\"specific_tags\"])]\n",
    "            page_lab = crf.predict(data_to_tag)\n",
    "            assert len(page_lab[0]) == len(page[\"offsets\"])\n",
    "            page.update({\"BET_tags\":page_lab[0]})\n",
    "        else:\n",
    "            page.update({\"BET_tags\":text2labelsTBE(page[\"offsets\"])})\n",
    "    return doc\n",
    "            \n",
    "threads = Pool(45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# parse all\n",
    "data2 = list()\n",
    "for ar in threads.imap_unordered(process_document, data):\n",
    "    data2.append(ar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#pickle.dump(data2, open(\"data/data.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# parse the references in a more json-like formar\n",
    "from code.support_functions import json_outputter\n",
    "_, refs, _ = json_outputter(data2, 40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(refs[10])"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:lb_main_analysis]",
   "language": "python",
   "name": "conda-env-lb_main_analysis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}