pycon-code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import time\n",
    "import numpy as np\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is the first review: bromwell high is a cartoon comedy . it ran at the same time as some other programs about school life  such as  teachers  . my   years in the teaching profession lead me to believe that bromwell high  s satire is much closer to reality than is  teachers  . the scramble to survive financially  the insightful students who can see right through their pathetic teachers  pomp  the pettiness of the whole situation  all remind me of the schools i knew and their students . when i saw the episode in which a student repeatedly tried to burn down the school  i immediately recalled . . . . . . . . . at . . . . . . . . . . high . a classic line inspector i  m here to sack one of your teachers . student welcome to bromwell high . i expect that many adults of my age think that bromwell high is far fetched . what a pity that it isn  t   \n",
      "\n",
      "This is the associated label: positive\n"
     ]
    }
   ],
   "source": [
    "# Read reviews and labels from the file\n",
    "reviews = map(lambda r: r[:-1], open('reviews.txt', 'r').readlines())\n",
    "labels = map(lambda l: l[:-1], open('labels.txt', 'r').readlines())\n",
    "\n",
    "print('This is the first review: {}\\n'.format(reviews[0]))\n",
    "print('This is the associated label: {}'.format(labels[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "total_words = Counter()\n",
    "total_labels = Counter()\n",
    "positive_words = Counter()\n",
    "negative_words = Counter()\n",
    "\n",
    "# Segregate words on the basis of positive or negative review\n",
    "for i in range(len(reviews)):\n",
    "    total_labels[labels[i]] += 1\n",
    "    for word in reviews[i].split(\" \"):\n",
    "        total_words[word] += 1\n",
    "        if labels[i] == 'positive':\n",
    "            positive_words[word] += 1\n",
    "        else:\n",
    "            negative_words[word] += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "vocab = set(total_words.keys())\n",
    "label_vocab = set(total_labels.keys())\n",
    "\n",
    "vocab_size = len(vocab)\n",
    "label_vocab_size = len(label_vocab)\n",
    "\n",
    "pn_ratios = Counter()\n",
    "\n",
    "# We are ignoring words which have a small word count\n",
    "# since they have significantly less impact as to how a review can be\n",
    "# positive or negative\n",
    "for word, count in total_words.most_common():\n",
    "    if(count < 100):\n",
    "        continue\n",
    "    pn_ratios[word] = positive_words[word] / float(negative_words[word] + 1)\n",
    "    \n",
    "# Here we find the ratio of the words present in positive reviews\n",
    "# to negative reviews, and find a different ratio for if the ratio is 1\n",
    "# add an offset to it, since the log of 1 is 0, which wouldn't be of lot of use\n",
    "# to neural network\n",
    "for word, ratio in pn_ratios.most_common():\n",
    "    if(ratio > 1):\n",
    "        pn_ratios[word] = np.log(ratio)\n",
    "        continue\n",
    "    pn_ratios[word] = np.log(ratio + 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('edie', 4.6913478822291435),\n",
       " ('paulie', 4.0775374439057197),\n",
       " ('felix', 3.1527360223636558),\n",
       " ('polanski', 2.8233610476132043),\n",
       " ('matthau', 2.8067217286092401),\n",
       " ('victoria', 2.6810215287142909),\n",
       " ('mildred', 2.6026896854443837),\n",
       " ('gandhi', 2.5389738710582761),\n",
       " ('flawless', 2.451005098112319),\n",
       " ('superbly', 2.2600254785752498),\n",
       " ('perfection', 2.1594842493533721),\n",
       " ('astaire', 2.1400661634962708),\n",
       " ('captures', 2.0386195471595809),\n",
       " ('voight', 2.0301704926730531),\n",
       " ('wonderfully', 2.0218960560332353),\n",
       " ('powell', 1.9783454248084671),\n",
       " ('brosnan', 1.9547990964725592),\n",
       " ('lily', 1.9203768470501485),\n",
       " ('bakshi', 1.9029851043382795),\n",
       " ('lincoln', 1.9014583864844796),\n",
       " ('refreshing', 1.8551812956655511),\n",
       " ('breathtaking', 1.8481124057791867),\n",
       " ('bourne', 1.8478489358790986),\n",
       " ('lemmon', 1.8458266904983307),\n",
       " ('delightful', 1.8002701588959635),\n",
       " ('flynn', 1.7996646487351682),\n",
       " ('andrews', 1.7764919970972666),\n",
       " ('homer', 1.7692866133759964),\n",
       " ('beautifully', 1.7626953362841438),\n",
       " ('soccer', 1.7578579175523736)]"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(pn_ratios.most_common())[0:30] # positive words have a higher ratio, and negative words have lower ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('boll', -4.0778152602708904),\n",
       " ('uwe', -3.9218753018711574),\n",
       " ('seagal', -3.3202501058581921),\n",
       " ('unwatchable', -3.0269848170580955),\n",
       " ('stinker', -2.9876839403711624),\n",
       " ('mst', -2.7753833211707972),\n",
       " ('incoherent', -2.7641396677532537),\n",
       " ('unfunny', -2.5545257844967644),\n",
       " ('waste', -2.4907515123361046),\n",
       " ('blah', -2.4475792789485),\n",
       " ('horrid', -2.3715779644809971),\n",
       " ('pointless', -2.3451073877136341),\n",
       " ('atrocious', -2.3187369339642556),\n",
       " ('redeeming', -2.2667790015910296),\n",
       " ('prom', -2.2601040980178784),\n",
       " ('drivel', -2.2476029585766928),\n",
       " ('lousy', -2.2118080125207054),\n",
       " ('worst', -2.1930856334332267),\n",
       " ('laughable', -2.172468615469592),\n",
       " ('awful', -2.1385076866397488),\n",
       " ('poorly', -2.1326133844207011),\n",
       " ('wasting', -2.1178155545614512),\n",
       " ('remotely', -2.111046881095167),\n",
       " ('existent', -2.0024805005437076),\n",
       " ('boredom', -1.9241486572738005),\n",
       " ('miserably', -1.9216610938019989),\n",
       " ('sucks', -1.9166645809588516),\n",
       " ('uninspired', -1.9131499212248517),\n",
       " ('lame', -1.9117232884159072),\n",
       " ('insult', -1.9085323769376259)]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(reversed(pn_ratios.most_common()))[0:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "input_layer = np.zeros((1, len(vocab)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "word2index = {}\n",
    "label2index = {}\n",
    "\n",
    "for i,word in enumerate(vocab):\n",
    "    word2index[word] = i\n",
    "    \n",
    "for i, label in enumerate(label_vocab):\n",
    "    label2index[label] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# We update the input layer, everytime a review is run through the neural network\n",
    "# so we will need to set the input layer to its initial state of all the node values\n",
    "# being zero.\n",
    "def update_input_layer(review):    \n",
    "    global layer_0\n",
    "\n",
    "    layer_0 *= 0\n",
    "    for word in review.split(\" \"):\n",
    "        layer_0[0][word2index[word]] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# Get the required number depending upon the sentiment exhibited by the label\n",
    "def get_number_for_label(label):\n",
    "    return 1 if label == 'positive' else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "class SentimentAnalysisNetwork:\n",
    "    def __init__(self, reviews, labels, output_nodes=1, hidden_nodes = 10, learning_rate = 0.1):\n",
    "        \"\"\"\n",
    "        Initialize the network by specifying the number of nodes and LR.\n",
    "        \"\"\"\n",
    "        self.input_nodes = vocab_size\n",
    "        self.hidden_nodes = hidden_nodes\n",
    "        self.output_nodes = output_nodes\n",
    "        self.learning_rate = learning_rate\n",
    "        self.input_layer = np.zeros((1, self.input_nodes))\n",
    "        self.weights_0_1 = np.zeros((self.input_nodes,self.hidden_nodes))\n",
    "        self.weights_1_2 = np.random.normal(\n",
    "            0.0, self.output_nodes**-0.5, (self.hidden_nodes, self.output_nodes)\n",
    "        )\n",
    "        \n",
    "    def update_input_layer(self,review):\n",
    "        \"\"\"\n",
    "        Set the input layer to be 0 after every review.\n",
    "        \"\"\"\n",
    "        self.input_layer *= 0\n",
    "        for word in review.split(\" \"):\n",
    "            if(word in word2index.keys()):\n",
    "                self.input_layer[0][word2index[word]] = 1\n",
    "                \n",
    "    def get_target_for_label(self,label):\n",
    "        \"\"\"\n",
    "        Returns a number given a label for a review.\n",
    "        \"\"\"\n",
    "        return 1 if label == 'positive' else 0\n",
    "        \n",
    "    def sigmoid(self,x):\n",
    "        \"\"\"\n",
    "        Returns the value when applied to a hypothesis function.\n",
    "        \"\"\"\n",
    "        return 1 / (1 + np.exp(-x))\n",
    "    \n",
    "    def sigmoid_output_2_derivative(self,output):\n",
    "        \"\"\"\n",
    "        Derivative of the hypothesis function.\n",
    "        \"\"\"\n",
    "        return output * (1 - output)\n",
    "    \n",
    "    def train(self, training_reviews, training_labels):\n",
    "        \"\"\"\n",
    "        Train the model by adjusting the hyper parameters.\n",
    "        Use forward propagation and backward propagation to train the model.\n",
    "        \"\"\"\n",
    "        correct_so_far = 0   \n",
    "        start = time.time()\n",
    "        \n",
    "        for i in range(len(training_reviews)):\n",
    "            review = training_reviews[i]\n",
    "            label = training_labels[i]\n",
    "\n",
    "            # Input Layer\n",
    "            self.update_input_layer(review)\n",
    "\n",
    "            # Hidden layer\n",
    "            hidden_layer = self.input_layer.dot(self.weights_0_1)\n",
    "\n",
    "            # Output layer\n",
    "            output_layer = self.sigmoid(hidden_layer.dot(self.weights_1_2))\n",
    "\n",
    "            ### BACKWARD PROPAGATION ###\n",
    "            # Output error\n",
    "            output_layer_error = output_layer - self.get_target_for_label(label)\n",
    "            output_layer_delta = (\n",
    "                output_layer_error * self.sigmoid_output_2_derivative(output_layer)\n",
    "            )\n",
    "\n",
    "            # Backpropagated error\n",
    "            hidden_layer_error = output_layer_delta.dot(self.weights_1_2.T)\n",
    "            hidden_layer_delta = hidden_layer_error\n",
    "\n",
    "            # Update the weights\n",
    "            self.weights_1_2 -= (\n",
    "                hidden_layer.T.dot(output_layer_delta) * self.learning_rate\n",
    "            )\n",
    "            self.weights_0_1 -= (\n",
    "                self.input_layer.T.dot(hidden_layer_delta) * self.learning_rate\n",
    "            )\n",
    "\n",
    "            if(np.abs(output_layer_error) < 0.5):\n",
    "                correct_so_far += 1\n",
    "            \n",
    "            reviews_per_second = i / float(time.time() - start)\n",
    "            \n",
    "            sys.stdout.write(\n",
    "                \"\\rProgress:\" + str(100 * i/float(len(training_reviews)))[:4] +\n",
    "                \"% Speed(reviews/sec):\" + str(reviews_per_second)[0:5] +\n",
    "                \" #Correct:\" + str(correct_so_far) +\n",
    "                \" #Trained:\" + str(i+1) + \" Training Accuracy:\" +\n",
    "                str(correct_so_far * 100 / float(i+1))[:4] + \"%\"\n",
    "            )\n",
    "            if(i % 2500 == 0):\n",
    "                continue\n",
    "    \n",
    "    def test(self, testing_reviews, testing_labels):\n",
    "        \"\"\"\n",
    "        Run the neural network with the parameters adjusted by training.\n",
    "        \"\"\"\n",
    "        correct = 0\n",
    "        start = time.time()\n",
    "        \n",
    "        for i in range(len(testing_reviews)):\n",
    "            pred = self.run(testing_reviews[i])\n",
    "            if(pred == testing_labels[i]):\n",
    "                correct += 1\n",
    "            \n",
    "            reviews_per_second = i / float(time.time() - start)\n",
    "            \n",
    "            sys.stdout.write(\n",
    "                \"\\rProgress:\" + str(100 * i/float(len(testing_reviews)))[:4] \\\n",
    "                + \"% Speed(reviews/sec):\" + str(reviews_per_second)[0:5] \\\n",
    "                + \"% #Correct:\" + str(correct) + \" #Tested:\" + str(i+1) +\n",
    "                \" Testing Accuracy:\" + str(correct * 100 / float(i+1))[:4] + \"%\"\n",
    "            )\n",
    "    \n",
    "    def run(self, review):\n",
    "        \"\"\"\n",
    "        Run the neural network with the parameters.\n",
    "        \"\"\"\n",
    "        self.update_input_layer(review.lower())\n",
    "        hidden_layer = self.input_layer.dot(self.weights_0_1)\n",
    "        layer_2 = self.sigmoid(hidden_layer.dot(self.weights_1_2))\n",
    "        return 'positive' if layer_2[0] > 0.5 else 'negative'   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "san = SentimentAnalysisNetwork(reviews[:-1000],labels[:-1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "#san.test(reviews[-1000:],labels[-1000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress:0.0% Speed(reviews/sec):0.0 #Correct:0 #Trained:1 Training Accuracy:0.0%\n",
      "Progress:10.4% Speed(reviews/sec):0.853 #Correct:1945 #Trained:2501 Training Accuracy:77.7%\n",
      "Progress:20.8% Speed(reviews/sec):0.207 #Correct:3994 #Trained:5001 Training Accuracy:79.8%\n",
      "Progress:31.2% Speed(reviews/sec):0.274 #Correct:6116 #Trained:7501 Training Accuracy:81.5%\n",
      "Progress:41.6% Speed(reviews/sec):0.329 #Correct:8272 #Trained:10001 Training Accuracy:82.7%\n",
      "Progress:52.0% Speed(reviews/sec):0.334 #Correct:10419 #Trained:12501 Training Accuracy:83.3%\n",
      "Progress:62.5% Speed(reviews/sec):0.228 #Correct:12561 #Trained:15001 Training Accuracy:83.7%\n",
      "Progress:72.9% Speed(reviews/sec):0.251 #Correct:14681 #Trained:17501 Training Accuracy:83.8%\n",
      "Progress:83.3% Speed(reviews/sec):0.271 #Correct:16862 #Trained:20001 Training Accuracy:84.3%\n",
      "Progress:93.7% Speed(reviews/sec):0.292 #Correct:19043 #Trained:22501 Training Accuracy:84.6%\n",
      "Progress:99.9% Speed(reviews/sec):0.304 #Correct:20365 #Trained:24000 Training Accuracy:84.8%"
     ]
    }
   ],
   "source": [
    "san.train(reviews[:-1000],labels[:-1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "notes"
    }
   },
   "outputs": [],
   "source": [
    "# Potential fitfalls of the network - go  through nano degree videos again!\n",
    "\n",
    "# 1. There are vocab words which are not present in the review, such as 'this movie is bad', now the word\n",
    "# good has a count of 0, and hence when it is multiplied with the weights it does not contribute anything, \n",
    "# so we can ignore all the words with word count as 0, which reduces the number of calculations significantly.\n",
    "\n",
    "# 2. You could also use NLP to see what would be the potential words of interests in a review, such as verbs\n",
    "# and adjectives and ignore names, pronouns or prepositions.\n",
    "\n",
    "# 3. Please dont feed the number of words present in a review, instead just take if the word appears just\n",
    "# to be it 1. So word count for a word appearing in a review is always one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress:99.5% Speed(reviews/sec):0.805% #Correct:173 #Tested:200 Testing Accuracy:86.5%"
     ]
    }
   ],
   "source": [
    "san.test(reviews[-200:],labels[-200:])"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}