Final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pylab as plt\n",
    "from scipy.stats.stats import pearsonr\n",
    "import nltk\n",
    "from nltk.tokenize import treebank\n",
    "from nltk.corpus import opinion_lexicon\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "import sys\n",
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "posW = opinion_lexicon.positive()\n",
    "negW = opinion_lexicon.negative()\n",
    "score = defaultdict(int)\n",
    "for w in posW:\n",
    "    score[w] = 1\n",
    "for w in negW:\n",
    "    score[w] = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "markers = {\"JJ\" : 0,\"JJR\" : 0,\"JJS\" : 0,\"NN\" : 1,\"NNP\" : 1,\"NNPS\" : 1,\"NNS\" : 1,\"PRP\" : 1,\"PRP$\" : 1,\"RB\" : 2,\n",
    "\"RBR\" : 2,\"RBS\" : 2,\"VB\" : 3,\"VBD\" : 3,\"VBG\" : 3,\"VBN\" : 3,\"VBP\" : 3,\"VBZ\" : 3} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_sentiment(text):\n",
    "    blob = TextBlob(text)\n",
    "    sent=0.0\n",
    "    for sentence in blob.sentences:\n",
    "        sent+=sentence.sentiment.polarity\n",
    "    sent /= len(blob.sentences)\n",
    "    return (sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_ari():\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_noun_count(tags):\n",
    "    val = 0\n",
    "    for t in tags:\n",
    "        tag = t[1]\n",
    "        if tag in markers.keys():\n",
    "            if tag[0] == 'N':\n",
    "                val += 1\n",
    "    return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_adj_count(review):\n",
    "    val = 0\n",
    "    for t in tags:\n",
    "        tag = t[1]\n",
    "        if tag in markers.keys():\n",
    "            if tag[0] == 'J':\n",
    "                val += 1\n",
    "    return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_verb_count(review):\n",
    "    val = 0\n",
    "    for t in tags:\n",
    "        tag = t[1]\n",
    "        if tag in markers.keys():\n",
    "            if tag[0] == 'N':\n",
    "                val += 1\n",
    "    return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from gensim import corpora, models\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "\n",
    "def do_LDA():\n",
    "    file = open('reviews.json')\n",
    "    reviews = []\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    en_stop = stopwords.words('english')\n",
    "    p_stemmer  = PorterStemmer()\n",
    "    c = 0\n",
    "    for l in file:\n",
    "        line = eval(l)\n",
    "        \n",
    "        c = c+1\n",
    "        if c == 1000:\n",
    "            break\n",
    "   \n",
    "        try:\n",
    "            tokens = tokenizer.tokenize(line['text'].lower())\n",
    "            stopped_tokens = [i for i in tokens if i not in en_stop ]\n",
    "            p_stemmer = PorterStemmer()\n",
    "            tokens = [p_stemmer.stem(i) for i in stopped_tokens]\n",
    "            reviews.append(tokens)\n",
    "        except UnicodeDecodeError:\n",
    "            print \"F\",\n",
    "        \n",
    "    dictionary = corpora.Dictionary(reviews)\n",
    "    \n",
    "    corpus = [dictionary.doc2bow(text) for text in reviews]\n",
    "    ldamodel = models.ldamodel.LdaModel(corpus, num_topics=5, id2word = dictionary, passes=10)\n",
    "\n",
    "    print ldamodel[corpus[0]]\n",
    "    return ldamodel, dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sriram/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:23: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F F"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-284-cb8de7f500ab>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mfile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'reviews.json'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mlda\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdictionary\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdo_LDA\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-283-f8d449be63b1>\u001b[0m in \u001b[0;36mdo_LDA\u001b[1;34m()\u001b[0m\n\u001b[0;32m     31\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     32\u001b[0m     \u001b[0mcorpus\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mdictionary\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdoc2bow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtext\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mreviews\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 33\u001b[1;33m     \u001b[0mldamodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mldamodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mLdaModel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_topics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mid2word\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdictionary\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpasses\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     34\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mldamodel\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/sriram/anaconda2/lib/python2.7/site-packages/gensim/models/ldamodel.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, corpus, num_topics, id2word, distributed, chunksize, passes, update_every, alpha, eta, decay, offset, eval_every, iterations, gamma_threshold, minimum_probability, random_state, ns_conf, minimum_phi_value, per_word_topics)\u001b[0m\n\u001b[0;32m    327\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLdaState\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_topics\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_terms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    328\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msstats\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgamma\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m100.\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1.\u001b[0m \u001b[1;33m/\u001b[0m \u001b[1;36m100.\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_topics\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_terms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 329\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexpElogbeta\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdirichlet_expectation\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msstats\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    330\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    331\u001b[0m         \u001b[1;31m# if a training corpus was provided, start estimating the model right away\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/sriram/anaconda2/lib/python2.7/site-packages/gensim/matutils.pyc\u001b[0m in \u001b[0;36mdirichlet_expectation\u001b[1;34m(alpha)\u001b[0m\n\u001b[0;32m    535\u001b[0m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpsi\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mpsi\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    536\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 537\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpsi\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mpsi\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnewaxis\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    538\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# keep the same precision as input\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    539\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "file = open('reviews.json')\n",
    "lda, dictionary = do_LDA()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make datasets here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sriram/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:26: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import RegexpTokenizer\n",
    "\n",
    "file = open('reviews.json')\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "en_stop = stopwords.words('english')\n",
    "p_stemmer  = PorterStemmer()\n",
    "dataset = []\n",
    "lenVsRating = {}\n",
    "dct = {}\n",
    "c = 0\n",
    "for l in file:\n",
    "    c = c+1\n",
    "    line = eval(l)\n",
    "    review = line['text']\n",
    "    user_id = line['user_id']\n",
    "    if user_id not in dct:\n",
    "        dct[user_id] = 0\n",
    "    if c >= 100000:\n",
    "        break\n",
    "    \n",
    "    try:\n",
    "        tokens = nltk.word_tokenize(review.lower())\n",
    "        text = nltk.Text(tokens)\n",
    "        tags = nltk.pos_tag(text)\n",
    "        \n",
    "        tokens = tokenizer.tokenize(line['text'].lower())\n",
    "        stopped_tokens = [i for i in tokens if i not in en_stop ]\n",
    "        p_stemmer = PorterStemmer()\n",
    "        tokens = [p_stemmer.stem(i) for i in stopped_tokens]\n",
    "         \n",
    "        \n",
    "        bow = dictionary.doc2bow(tokens)\n",
    "        \n",
    "        # Features are nouns, length, sentiment, ratingz\n",
    "        vec = [1.0]\n",
    "        vec.append(get_noun_count(tags))\n",
    "        vec.append(get_verb_count(tags))\n",
    "        vec.append(get_adj_count(tags))\n",
    "        vec.append(len(review))\n",
    "        vec.append(get_sentiment(review))\n",
    "        \n",
    "        ldaVec = [0]*15\n",
    "        for val in lda[bow]:\n",
    "            ldaVec[val[0]] = val[1]\n",
    "        vec = vec + ldaVec\n",
    "        \n",
    "        vec.append(float(line['stars']))        \n",
    "        dataset.append(vec) \n",
    "        \n",
    "    except UnicodeDecodeError:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "675.78019586507071"
      ]
     },
     "execution_count": 300,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(dataset[:, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(91170, 22)\n"
     ]
    }
   ],
   "source": [
    "dataset = np.array(dataset)\n",
    "print dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8830"
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "100000-91170"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17\n",
      "  18  19  20  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35\n",
      "  36  37  38  39  40  41  42  43  44  45  46  47  48  49  50  51  52  54\n",
      "  55  56  62  65  66  68  70  71  83 103]\n",
      "[ 3.75        3.          3.66666667  3.77777778  3.7755102   3.95238095\n",
      "  3.52272727  3.41860465  3.43902439  3.47916667  3.2         3.35483871\n",
      "  3.38095238  3.42857143  3.35483871  3.55555556  3.78571429  3.34782609\n",
      "  3.52631579  3.76        3.41176471  3.75        3.26315789  2.42857143\n",
      "  3.          3.21428571  2.92307692  1.85714286  3.35714286  3.85714286\n",
      "  3.22222222  2.71428571  4.25        3.16666667  3.4         3.33333333\n",
      "  4.          2.5         2.8         1.          2.88888889  3.2         1.75\n",
      "  3.33333333  3.66666667  3.33333333  3.          3.          4.          3.\n",
      "  4.          3.          3.          3.5         3.5         3.66666667\n",
      "  4.          2.          1.          3.          5.          5.          2.\n",
      "  1.        ]\n",
      "(-0.2885441671050909, 0.020762488393339475)\n"
     ]
    }
   ],
   "source": [
    "k = 2\n",
    "dt = {}\n",
    "\n",
    "for key, value in zip(dataset[:, 2], dataset[:, -1]):\n",
    "    \n",
    "    if int(key/k) not in dt:\n",
    "        dt[int(key/k)] = []\n",
    "    dt[int(key/k)].append(value)\n",
    "        \n",
    "keys = sorted(dt.keys())\n",
    "x = []\n",
    "y = []\n",
    "for key in keys:\n",
    "    x.append(key)\n",
    "#     print len(dt[key])\n",
    "    y.append((np.mean(dt[key])))\n",
    "    \n",
    "x = np.array(x)\n",
    "y = np.array(y)\n",
    "print x\n",
    "print y\n",
    "plt.scatter(np.array(x)*k, y, s = 150, marker = '+', color = 'navy')\n",
    "plt.xlabel('Number of nouns')\n",
    "plt.ylabel('Rating')\n",
    "print pearsonr(x, y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X = dataset[:, :-1]\n",
    "Y = dataset[:, -1]\n",
    "\n",
    "idxs = np.random.choice(len(X), int(0.2*len(X)))\n",
    "n_idx = [i for i in range(len(X)) if i not in idxs]\n",
    "\n",
    "trainX = X[idxs]\n",
    "trainY = Y[idxs]\n",
    "\n",
    "testX  = X[n_idx]\n",
    "testY = Y[n_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.47540983607 3.44652406417\n"
     ]
    }
   ],
   "source": [
    "print np.mean(trainY), np.mean(testY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train linear regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.828877005348\n",
      "-0.0112981719017\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "rgr = ExtraTreesRegressor(n_estimators=500, max_depth=5)\n",
    "rgr.fit(trainX, trainY)\n",
    "\n",
    "prediction = rgr.predict(testX)\n",
    "prediction = np.round(prediction)\n",
    "for i in range(len(prediction)):\n",
    "    if prediction[i] < 1:\n",
    "        prediction[i] = 1\n",
    "    if prediction[i] > 5:\n",
    "        prediction[i] = 5\n",
    "print np.sum(np.abs(prediction-testY))/len(testX)\n",
    "print r2_score(testY, [3.59]*len(testY))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Topic vs MAE\n",
    "\n",
    "\n",
    "x = [1, 2, 3, 4, 5, 6, 7, 8]\n",
    "y = [0.87, 0.854, 0.835, 0.83, 0.827, 0.86, 0.881, 0.874]\n",
    "\n",
    "plt.plot(x, y, linewidth = 1)\n",
    "plt.xlabel('Number of topics')\n",
    "plt.ylabel('MAE')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nvec = np.array(dataset)\n",
    "\n",
    "kys = lenVsRating.keys()\n",
    "kys = sorted(kys)\n",
    "\n",
    "x = []\n",
    "y = []\n",
    "for ky in kys:\n",
    "    x.append(ky)\n",
    "    y.append(np.round(np.mean(lenVsRating[ky])))\n",
    "\n",
    "\n",
    "plt.scatter(np.array(x)*k, y, s = 150, color = 'navy', marker = '+')\n",
    "plt.xlabel('Number of nouns used')\n",
    "plt.ylabel('Rating')\n",
    "print pearsonr(x, y)\n",
    "plt.show()\n",
    "\n",
    "\n",
    "# Len - -0.6262\n",
    "# Verb - 0.561\n",
    "# Adjective - 0.255\n",
    "# Noun - -0.193\n",
    "# Punctuation - 0.122\n",
    "# Average word length - 0.035\n",
    "# Average sentence length - 0.067\n",
    "# Sentiment analysis - 0.61"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}