lstm_crf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fbe4f9585b0>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.autograd as autograd\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "\n",
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def argmax(vec):\n",
    "    # return the argmax as a python int\n",
    "    _, idx = torch.max(vec, 1)\n",
    "    return idx.item()\n",
    "\n",
    "\n",
    "def prepare_sequence(seq, to_ix):\n",
    "    idxs = [to_ix[w] for w in seq]\n",
    "    return torch.LongTensor(idxs)\n",
    "\n",
    "\n",
    "# Compute log sum exp in a numerically stable way for the forward algorithm\n",
    "def log_sum_exp(vec):\n",
    "    max_score = vec[0, argmax(vec)]\n",
    "    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])\n",
    "    return max_score + \\\n",
    "        torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class BiLSTM_CRF(nn.Module):\n",
    "\n",
    "    def __init__(self, vocab_size, tag_to_ix, embedding_dim, hidden_dim):\n",
    "        super(BiLSTM_CRF, self).__init__()\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.vocab_size = vocab_size\n",
    "        self.tag_to_ix = tag_to_ix\n",
    "        self.tagset_size = len(tag_to_ix)\n",
    "\n",
    "        self.word_embeds = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim // 2,\n",
    "                            num_layers=1, bidirectional=True)\n",
    "\n",
    "        # Maps the output of the LSTM into tag space.\n",
    "        self.hidden2tag = nn.Linear(hidden_dim, self.tagset_size)\n",
    "\n",
    "        # Matrix of transition parameters.  Entry i,j is the score of\n",
    "        # transitioning *to* i *from* j.\n",
    "        self.transitions = nn.Parameter(\n",
    "            torch.randn(self.tagset_size, self.tagset_size))\n",
    "\n",
    "        # These two statements enforce the constraint that we never transfer\n",
    "        # to the start tag and we never transfer from the stop tag\n",
    "        self.transitions.data[tag_to_ix[START_TAG], :] = -10000\n",
    "        self.transitions.data[:, tag_to_ix[STOP_TAG]] = -10000\n",
    "\n",
    "        self.hidden = self.init_hidden()\n",
    "\n",
    "    def init_hidden(self):\n",
    "        return (torch.randn(2, 1, self.hidden_dim // 2),\n",
    "                torch.randn(2, 1, self.hidden_dim // 2))\n",
    "\n",
    "    def _forward_alg(self, feats):\n",
    "        # Do the forward algorithm to compute the partition function\n",
    "        init_alphas = torch.full((1, self.tagset_size), -10000.)\n",
    "        # START_TAG has all of the score.\n",
    "        init_alphas[0][self.tag_to_ix[START_TAG]] = 0.\n",
    "\n",
    "        # Wrap in a variable so that we will get automatic backprop\n",
    "        forward_var = init_alphas\n",
    "\n",
    "        # Iterate through the sentence\n",
    "        for feat in feats:\n",
    "            alphas_t = []  # The forward tensors at this timestep\n",
    "            for next_tag in range(self.tagset_size):\n",
    "                # broadcast the emission score: it is the same regardless of\n",
    "                # the previous tag\n",
    "                emit_score = feat[next_tag].view(\n",
    "                    1, -1).expand(1, self.tagset_size)\n",
    "                # the ith entry of trans_score is the score of transitioning to\n",
    "                # next_tag from i\n",
    "                trans_score = self.transitions[next_tag].view(1, -1)\n",
    "                # The ith entry of next_tag_var is the value for the\n",
    "                # edge (i -> next_tag) before we do log-sum-exp\n",
    "                next_tag_var = forward_var + trans_score + emit_score\n",
    "                # The forward variable for this tag is log-sum-exp of all the\n",
    "                # scores.\n",
    "                alphas_t.append(log_sum_exp(next_tag_var).view(1))\n",
    "            forward_var = torch.cat(alphas_t).view(1, -1)\n",
    "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
    "        alpha = log_sum_exp(terminal_var)\n",
    "        return alpha\n",
    "\n",
    "    def _get_lstm_features(self, sentence):\n",
    "        self.hidden = self.init_hidden()\n",
    "        embeds = self.word_embeds(sentence).view(len(sentence), 1, -1)\n",
    "        lstm_out, self.hidden = self.lstm(embeds, self.hidden)\n",
    "        lstm_out = lstm_out.view(len(sentence), self.hidden_dim)\n",
    "        lstm_feats = self.hidden2tag(lstm_out)\n",
    "        return lstm_feats\n",
    "\n",
    "    def _score_sentence(self, feats, tags):\n",
    "        # Gives the score of a provided tag sequence\n",
    "        score = torch.zeros(1)\n",
    "        tags = torch.cat([torch.tensor([self.tag_to_ix[START_TAG]], dtype=torch.long), tags])\n",
    "        for i, feat in enumerate(feats):\n",
    "            score = score + \\\n",
    "                self.transitions[tags[i + 1], tags[i]] + feat[tags[i + 1]]\n",
    "        score = score + self.transitions[self.tag_to_ix[STOP_TAG], tags[-1]]\n",
    "        return score\n",
    "\n",
    "    def _viterbi_decode(self, feats):\n",
    "        backpointers = []\n",
    "\n",
    "        # Initialize the viterbi variables in log space\n",
    "        init_vvars = torch.full((1, self.tagset_size), -10000.)\n",
    "        init_vvars[0][self.tag_to_ix[START_TAG]] = 0\n",
    "\n",
    "        # forward_var at step i holds the viterbi variables for step i-1\n",
    "        forward_var = init_vvars\n",
    "        for feat in feats:\n",
    "            bptrs_t = []  # holds the backpointers for this step\n",
    "            viterbivars_t = []  # holds the viterbi variables for this step\n",
    "\n",
    "            for next_tag in range(self.tagset_size):\n",
    "                # next_tag_var[i] holds the viterbi variable for tag i at the\n",
    "                # previous step, plus the score of transitioning\n",
    "                # from tag i to next_tag.\n",
    "                # We don't include the emission scores here because the max\n",
    "                # does not depend on them (we add them in below)\n",
    "                next_tag_var = forward_var + self.transitions[next_tag]\n",
    "                best_tag_id = argmax(next_tag_var)\n",
    "                bptrs_t.append(best_tag_id)\n",
    "                viterbivars_t.append(next_tag_var[0][best_tag_id].view(1))\n",
    "            # Now add in the emission scores, and assign forward_var to the set\n",
    "            # of viterbi variables we just computed\n",
    "            forward_var = (torch.cat(viterbivars_t) + feat).view(1, -1)\n",
    "            backpointers.append(bptrs_t)\n",
    "\n",
    "        # Transition to STOP_TAG\n",
    "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
    "        best_tag_id = argmax(terminal_var)\n",
    "        path_score = terminal_var[0][best_tag_id]\n",
    "\n",
    "        # Follow the back pointers to decode the best path.\n",
    "        best_path = [best_tag_id]\n",
    "        for bptrs_t in reversed(backpointers):\n",
    "            best_tag_id = bptrs_t[best_tag_id]\n",
    "            best_path.append(best_tag_id)\n",
    "        # Pop off the start tag (we dont want to return that to the caller)\n",
    "        start = best_path.pop()\n",
    "        assert start == self.tag_to_ix[START_TAG]  # Sanity check\n",
    "        best_path.reverse()\n",
    "        return path_score, best_path\n",
    "\n",
    "    def neg_log_likelihood(self, sentence, tags):\n",
    "        feats = self._get_lstm_features(sentence)\n",
    "        forward_score = self._forward_alg(feats)\n",
    "        gold_score = self._score_sentence(feats, tags)\n",
    "        return forward_score - gold_score\n",
    "\n",
    "    def forward(self, sentence):  # dont confuse this with _forward_alg above.\n",
    "        # Get the emission scores from the BiLSTM\n",
    "        lstm_feats = self._get_lstm_features(sentence)\n",
    "\n",
    "        # Find the best path, given the features.\n",
    "        score, tag_seq = self._viterbi_decode(lstm_feats)\n",
    "        return score, tag_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "input.size(-1) must be equal to input_size. Expected 5, got 250",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-1422c7c07f25>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneg_log_likelihood\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-26-e9bc8c0115e1>\u001b[0m in \u001b[0;36mneg_log_likelihood\u001b[0;34m(self, sentence, tags)\u001b[0m\n\u001b[1;32m    125\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mneg_log_likelihood\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m         \u001b[0mfeats\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_lstm_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    128\u001b[0m         \u001b[0mforward_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_alg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeats\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m         \u001b[0mgold_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_score_sentence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-26-e9bc8c0115e1>\u001b[0m in \u001b[0;36m_get_lstm_features\u001b[0;34m(self, sentence)\u001b[0m\n\u001b[1;32m     66\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m         \u001b[0membeds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mword_embeds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m         \u001b[0mlstm_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m         \u001b[0mlstm_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlstm_out\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_dim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0mlstm_feats\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden2tag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlstm_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/torch/nn/modules/rnn.pyc\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    176\u001b[0m             \u001b[0mflat_weight\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_forward_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m         func = self._backend.RNN(\n\u001b[1;32m    180\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/torch/nn/modules/rnn.pyc\u001b[0m in \u001b[0;36mcheck_forward_args\u001b[0;34m(self, input, hidden, batch_sizes)\u001b[0m\n\u001b[1;32m    128\u001b[0m             raise RuntimeError(\n\u001b[1;32m    129\u001b[0m                 'input.size(-1) must be equal to input_size. Expected {}, got {}'.format(\n\u001b[0;32m--> 130\u001b[0;31m                     self.input_size, input.size(-1)))\n\u001b[0m\u001b[1;32m    131\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mis_input_packed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: input.size(-1) must be equal to input_size. Expected 5, got 250"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "START_TAG = \"<START>\"\n",
    "STOP_TAG = \"<STOP>\"\n",
    "PADDING = \"<PAD>\"\n",
    "# max sentence size\n",
    "max_len = 50\n",
    "\n",
    "EMBEDDING_DIM = 5\n",
    "HIDDEN_DIM = 4\n",
    "\n",
    "batch_size = 15\n",
    "\n",
    "train_file_x = 'data/restaurants_trial_x.txt'\n",
    "train_file_y = 'data/restaurants_trial_y.txt'\n",
    "\n",
    "def make_list(path):\n",
    "    f = open(path)\n",
    "    return [x.split() for x in f]\n",
    "\n",
    "train_x = make_list(train_file_x)\n",
    "# print('max len', max(map(len, train_x)))\n",
    "train_y = make_list(train_file_y)\n",
    "# training_data = list(zip(train_x, train_y))\n",
    "\n",
    "# add padding\n",
    "for i in xrange(len(train_x)):\n",
    "    for _ in xrange(max_len-len(train_x[i])):\n",
    "        train_x[i].append('<PAD>')\n",
    "        train_y[i].append('0')\n",
    "\n",
    "word_to_ix = {}\n",
    "for sentence in train_x:\n",
    "    for word in sentence:\n",
    "        if word not in word_to_ix:\n",
    "            word_to_ix[word] = len(word_to_ix)\n",
    "            \n",
    "# make train data tensors\n",
    "for i in xrange(len(training_data)):\n",
    "    train_x[i] = map(lambda x: word_to_ix[x], train_x[i])\n",
    "    train_y[i] = map(int, train_y[i])\n",
    "\n",
    "train_x = torch.LongTensor(train_x)\n",
    "train_y = torch.LongTensor(train_y)\n",
    "\n",
    "# print(\"shape train_x \", train_x.shape)\n",
    "training_data = zip(train_x, train_y)\n",
    "\n",
    "tag_to_ix = {\"0\": 0, \"1\": 1, START_TAG: 2, STOP_TAG: 3}\n",
    "\n",
    "model = BiLSTM_CRF(len(word_to_ix), tag_to_ix, EMBEDDING_DIM, HIDDEN_DIM)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-4)\n",
    "\n",
    "for epoch in range(1):\n",
    "    batch_id = 0\n",
    "    start = 0\n",
    "    end = 0\n",
    "    \n",
    "    loss_val = 0\n",
    "    \n",
    "    while(batch_id < int(math.ceil(len(training_data)/batch_size))):\n",
    "        start = batch_id * batch_size\n",
    "        end = min(start + batch_size, len(training_data))\n",
    "        \n",
    "        batch_x = train_x[start:end]\n",
    "        batch_y = train_y[start:end]\n",
    "        \n",
    "        model.zero_grad()\n",
    "        loss = model.neg_log_likelihood(batch_x, batch_y)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        loss_val += loss.tolist()[0]\n",
    "        batch_id += 1\n",
    "    \n",
    "    print(\"Loss at epoch %d\" % (epoch + 1), loss_val)\n",
    "    \n",
    "#     for sentence, tags in training_data:\n",
    "#         model.zero_grad()\n",
    "\n",
    "#         # Step 2. Get our inputs ready for the network, that is,\n",
    "#         # turn them into Tensors of word indices.\n",
    "#         sentence_in = prepare_sequence(sentence, word_to_ix)\n",
    "#         targets = torch.LongTensor([tag_to_ix[t] for t in tags])\n",
    "\n",
    "#         # Step 3. Run our forward pass.\n",
    "# #         print(sentence_in, targets)\n",
    "#         loss = model.neg_log_likelihood(sentence_in, targets)\n",
    "\n",
    "#         # Step 4. Compute the loss, gradients, and update the parameters by\n",
    "#         # calling optimizer.step()\n",
    "#         loss.backward()\n",
    "#         optimizer.step()\n",
    "\n",
    "# Check predictions after training\n",
    "# with torch.no_grad():\n",
    "#     precheck_sent = prepare_sequence(training_data[0][0], word_to_ix)\n",
    "#     print(model(precheck_sent))\n",
    "# We got it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "START_TAG = \"<START>\"\n",
    "STOP_TAG = \"<STOP>\"\n",
    "EMBEDDING_DIM = 5\n",
    "HIDDEN_DIM = 4\n",
    "\n",
    "# Make up some training data\n",
    "training_data = [(\n",
    "    \"the wall street journal reported today that apple corporation made money\".split(),\n",
    "    \"B I I I O O O B I O O\".split()\n",
    "), (\n",
    "    \"georgia tech is a university in georgia\".split(),\n",
    "    \"B I O O O O B\".split()\n",
    ")]\n",
    "\n",
    "word_to_ix = {}\n",
    "for sentence, tags in training_data:\n",
    "    for word in sentence:\n",
    "        if word not in word_to_ix:\n",
    "            word_to_ix[word] = len(word_to_ix)\n",
    "\n",
    "tag_to_ix = {\"B\": 0, \"I\": 1, \"O\": 2, START_TAG: 3, STOP_TAG: 4}\n",
    "\n",
    "model = BiLSTM_CRF(len(word_to_ix), tag_to_ix, EMBEDDING_DIM, HIDDEN_DIM)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-4)\n",
    "\n",
    "# Check predictions before training\n",
    "with torch.no_grad():\n",
    "    precheck_sent = prepare_sequence(training_data[0][0], word_to_ix)\n",
    "    precheck_tags = torch.tensor([tag_to_ix[t] for t in training_data[0][1]], dtype=torch.long)\n",
    "    print(model(precheck_sent))\n",
    "\n",
    "# Make sure prepare_sequence from earlier in the LSTM section is loaded\n",
    "for epoch in range(\n",
    "        300):  # again, normally you would NOT do 300 epochs, it is toy data\n",
    "    for sentence, tags in training_data:\n",
    "        # Step 1. Remember that Pytorch accumulates gradients.\n",
    "        # We need to clear them out before each instance\n",
    "        model.zero_grad()\n",
    "\n",
    "        # Step 2. Get our inputs ready for the network, that is,\n",
    "        # turn them into Tensors of word indices.\n",
    "        sentence_in = prepare_sequence(sentence, word_to_ix)\n",
    "        targets = torch.tensor([tag_to_ix[t] for t in tags], dtype=torch.long)\n",
    "\n",
    "        # Step 3. Run our forward pass.\n",
    "        loss = model.neg_log_likelihood(sentence_in, targets)\n",
    "\n",
    "        # Step 4. Compute the loss, gradients, and update the parameters by\n",
    "        # calling optimizer.step()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "# Check predictions after training\n",
    "with torch.no_grad():\n",
    "    precheck_sent = prepare_sequence(training_data[0][0], word_to_ix)\n",
    "    print(model(precheck_sent))\n",
    "# We got it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}