sas2016_spacy_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Follow the instructions here to install spacy and be able to run the commands\n",
    "# given below:\n",
    "#    https://spacy.io/docs#getting-started"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text = u\"Pierre Vinken, 61 years old, will join the board as a nonexecutive director Nov. 29. Mr. Vinken is chairman of Elsevier N.V., the Dutch publishing group. Rudolph Agnew, 55 years old and former chairman of Consolidated Gold Fields PLC, was named a director of this British industrial conglomerate.\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "doc = nlp(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pierre Vinken, 61 years old, will join the board as a nonexecutive director Nov. 29.\n",
      "Mr. Vinken is chairman of Elsevier N.V., the Dutch publishing group.\n",
      "Rudolph Agnew, 55 years old and former chairman of Consolidated Gold Fields PLC, was named a director of this British industrial conglomerate.\n"
     ]
    }
   ],
   "source": [
    "# Show the detected sentences.\n",
    "for sent in doc.sents:\n",
    "    print (sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Pierre\n",
      "1 Vinken\n",
      "2 ,\n",
      "3 61\n",
      "4 years\n",
      "5 old\n",
      "6 ,\n",
      "7 will\n",
      "8 join\n",
      "9 the\n",
      "10 board\n",
      "11 as\n",
      "12 a\n",
      "13 nonexecutive\n",
      "14 director\n",
      "15 Nov.\n",
      "16 29\n",
      "17 .\n",
      "18 Mr.\n",
      "19 Vinken\n",
      "20 is\n",
      "21 chairman\n",
      "22 of\n",
      "23 Elsevier\n",
      "24 N.V.\n",
      "25 ,\n",
      "26 the\n",
      "27 Dutch\n",
      "28 publishing\n",
      "29 group\n",
      "30 .\n",
      "31 Rudolph\n",
      "32 Agnew\n",
      "33 ,\n",
      "34 55\n",
      "35 years\n",
      "36 old\n",
      "37 and\n",
      "38 former\n",
      "39 chairman\n",
      "40 of\n",
      "41 Consolidated\n",
      "42 Gold\n",
      "43 Fields\n",
      "44 PLC\n",
      "45 ,\n",
      "46 was\n",
      "47 named\n",
      "48 a\n",
      "49 director\n",
      "50 of\n",
      "51 this\n",
      "52 British\n",
      "53 industrial\n",
      "54 conglomerate\n",
      "55 .\n"
     ]
    }
   ],
   "source": [
    "# Show the detected tokens.\n",
    "index = 0\n",
    "for tok in doc:\n",
    "    print (index, tok)\n",
    "    index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.615261651339 chairman <-> director\n",
      "0.35017640956 chairman <-> conglomerate\n",
      "----\n",
      "0.311771623351 group <-> director\n",
      "0.322598796783 group <-> conglomerate\n"
     ]
    }
   ],
   "source": [
    "# Show word similarities.\n",
    "# 29 is \"group\", 39 is \"chairman\", 49 is \"director\", 54 is \"conglomerate\"\n",
    "print (doc[39].similarity(doc[49]), \"chairman <-> director\")\n",
    "print (doc[39].similarity(doc[54]), \"chairman <-> conglomerate\")\n",
    "print (\"----\")\n",
    "print (doc[29].similarity(doc[49]), \"group <-> director\")\n",
    "print (doc[29].similarity(doc[54]), \"group <-> conglomerate\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "years year\n",
      "is be\n",
      "years year\n",
      "was be\n",
      "named name\n"
     ]
    }
   ],
   "source": [
    "# Show the lemmas of each of the tokens, if not the same as the orthographic.\n",
    "# form.\n",
    "for tok in doc:\n",
    "    if (tok.lower_ != tok.lemma_):\n",
    "        print (tok, tok.lemma_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-3.0678977966308594 .\n",
      "-3.4549596309661865 ,\n",
      "-3.528766632080078 the\n",
      "-3.92978835105896 a\n",
      "-4.113108158111572 and\n",
      "-4.27587366104126 of\n",
      "-4.457748889923096 is\n",
      "-5.252320289611816 was\n",
      "-5.36181640625 this\n",
      "-5.53448486328125 as\n",
      "-6.199834823608398 will\n",
      "-7.368987560272217 years\n",
      "-7.845602989196777 old\n",
      "-8.761202812194824 group\n",
      "-9.715755462646484 board\n",
      "-9.882078170776367 join\n",
      "-10.133160591125488 named\n",
      "-10.352326393127441 British\n",
      "-10.373620986938477 former\n",
      "-10.547660827636719 Mr.\n",
      "-10.910798072814941 Gold\n",
      "-11.282051086425781 29\n",
      "-11.463842391967773 director\n",
      "-11.507254600524902 industrial\n",
      "-11.646064758300781 55\n",
      "-11.688076972961426 Dutch\n",
      "-12.308273315429688 publishing\n",
      "-12.375371932983398 Nov.\n",
      "-12.898519515991211 61\n",
      "-13.167049407958984 Pierre\n",
      "-13.772281646728516 Fields\n",
      "-13.928001403808594 chairman\n",
      "-14.147188186645508 Rudolph\n",
      "-14.507896423339844 conglomerate\n",
      "-15.370661735534668 PLC\n",
      "-16.369218826293945 Consolidated\n",
      "-16.572589874267578 Agnew\n",
      "-16.884357452392578 Elsevier\n",
      "-19.190162658691406 N.V.\n",
      "-19.502029418945312 Vinken\n",
      "-19.502029418945312 nonexecutive\n"
     ]
    }
   ],
   "source": [
    "# Rank the words from most probable to least.\n",
    "word_probs = set([(tok.prob, tok.orth_) for tok in doc])\n",
    "sorted_word_probs = sorted(word_probs,key=lambda tup: tup[0])\n",
    "sorted_word_probs.reverse()\n",
    "for prob,word in sorted_word_probs:\n",
    "    print (prob, word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get the first sentence.\n",
    "s1 = next(doc.sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pierre Xxxxx\n",
      "Vinken Xxxxx\n",
      ", ,\n",
      "61 dd\n",
      "years xxxx\n",
      "old xxx\n",
      ", ,\n",
      "will xxxx\n",
      "join xxxx\n",
      "the xxx\n",
      "board xxxx\n",
      "as xx\n",
      "a x\n",
      "nonexecutive xxxx\n",
      "director xxxx\n",
      "Nov. Xxx.\n",
      "29 dd\n",
      ". .\n"
     ]
    }
   ],
   "source": [
    "# Show the shapes of the tokens.\n",
    "for tok in s1:\n",
    "    print (tok, tok.shape_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROPN Pierre\n",
      "PROPN Vinken\n",
      "PUNCT ,\n",
      "NUM 61\n",
      "NOUN years\n",
      "ADJ old\n",
      "PUNCT ,\n",
      "VERB will\n",
      "VERB join\n",
      "DET the\n",
      "NOUN board\n",
      "ADP as\n",
      "DET a\n",
      "ADJ nonexecutive\n",
      "NOUN director\n",
      "PROPN Nov.\n",
      "NUM 29\n",
      "PUNCT .\n"
     ]
    }
   ],
   "source": [
    "# Show the part-of-speech tags.\n",
    "for tok in s1:\n",
    "    print (tok.pos_,tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROPN NNP Pierre\n",
      "PROPN NNP Vinken\n",
      "PUNCT , ,\n",
      "NUM CD 61\n",
      "NOUN NNS years\n",
      "ADJ JJ old\n",
      "PUNCT , ,\n",
      "VERB MD will\n",
      "VERB VB join\n",
      "DET DT the\n",
      "NOUN NN board\n",
      "ADP IN as\n",
      "DET DT a\n",
      "ADJ JJ nonexecutive\n",
      "NOUN NN director\n",
      "PROPN NNP Nov.\n",
      "NUM CD 29\n",
      "PUNCT . .\n"
     ]
    }
   ],
   "source": [
    "# Fine-grained POS tags too.\n",
    "for tok in s1:\n",
    "    print (tok.pos_,tok.tag_,tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NP Pierre Vinken\n",
      "NP the board\n",
      "NP a nonexecutive director\n",
      "NP Mr. Vinken\n",
      "NP chairman\n",
      "NP Elsevier N.V.\n",
      "NP Rudolph Agnew\n",
      "NP Consolidated Gold Fields PLC\n",
      "NP this British industrial conglomerate\n"
     ]
    }
   ],
   "source": [
    "# Noun-phrase chunks.\n",
    "for chunk in doc.noun_chunks:\n",
    "    print (chunk.label_, chunk.orth_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PERSON Pierre Vinken\n",
      "DATE 61 years old\n",
      "DATE Nov. 29\n",
      "PERSON Vinken\n",
      "ORG Elsevier N.V.\n",
      "NORP Dutch\n",
      "PERSON Rudolph Agnew\n",
      "DATE 55 years old\n",
      "ORG Consolidated Gold Fields PLC\n",
      "NORP British\n"
     ]
    }
   ],
   "source": [
    "# Named entities.\n",
    "for ent in doc.ents:\n",
    "    print (ent.label_, ent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pierre --compound--> Vinken\n",
      "Vinken --nsubj--> join\n",
      ", --punct--> Vinken\n",
      "61 --nummod--> years\n",
      "years --npadvmod--> old\n",
      "old --amod--> Vinken\n",
      ", --punct--> Vinken\n",
      "will --aux--> join\n",
      "join --ROOT--> join\n",
      "the --det--> board\n",
      "board --dobj--> join\n",
      "as --prep--> join\n",
      "a --det--> director\n",
      "nonexecutive --amod--> director\n",
      "director --pobj--> as\n",
      "Nov. --npadvmod--> join\n",
      "29 --nummod--> Nov.\n",
      ". --punct--> join\n"
     ]
    }
   ],
   "source": [
    "# Show the syntactic head for each token.\n",
    "for tok in s1:\n",
    "    print (tok, (\"--%s-->\" % tok.dep_), tok.head.orth_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Show the path from a token to the root. Adapted from spaCy docs.\n",
    "def dependency_labels_to_root(token):\n",
    "    '''Walk up the syntactic tree, collecting the arc labels.'''\n",
    "    dep_labels = []\n",
    "    while token.head is not token:\n",
    "        dependency = token.orth_+ \" \" +(\"--%s-->\" % token.dep_)+\" \" + token.head.orth_\n",
    "        dep_labels.append(dependency)\n",
    "        token = token.head\n",
    "    return dep_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pierre Vinken, 61 years old, will join the board as a nonexecutive director Nov. 29.\n"
     ]
    }
   ],
   "source": [
    "# Show the first sentence again, as a reminder.\n",
    "print (s1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "years --npadvmod--> old\n",
      "old --amod--> Vinken\n",
      "Vinken --nsubj--> join\n"
     ]
    }
   ],
   "source": [
    "# Path from \"years\"\n",
    "for dep in dependency_labels_to_root(s1[4]):\n",
    "    print (dep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nonexecutive --amod--> director\n",
      "director --pobj--> as\n",
      "as --prep--> join\n"
     ]
    }
   ],
   "source": [
    "# Path from \"nonexecutive\"\n",
    "for dep in dependency_labels_to_root(s1[13]):\n",
    "    print (dep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}