Movie_Classification_bkp.ipynb copy

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sp\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import defaultdict\n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "from lsh import clsh, jlsh, generateSamples, findNeighborsBrute, recall\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def filterLen(docs, minlen):\n",
    "    r\"\"\" filter out terms that are too short. \n",
    "    docs is a list of lists, each inner list is a document represented as a list of words\n",
    "    minlen is the minimum length of the word to keep\n",
    "    \"\"\"\n",
    "    return [ [t for t in d if len(t) >= minlen ] for d in docs ]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def getTestDocs():\n",
    "    with open(\"test.dat\", \"r\") as fh:\n",
    "        test_lines = fh.readlines()\n",
    "\n",
    "    \n",
    "    test_documents =  [l.split()[1:] for l in test_lines]\n",
    "    return test_documents\n",
    "\n",
    "\n",
    "def getTrainDocsSplit(testpercentage=0.10):\n",
    "    # open docs file and read its lines\n",
    "    with open(\"train.dat\", \"r\") as fh:\n",
    "        lines = fh.readlines()\n",
    "\n",
    "\n",
    "    size = len(lines)\n",
    "    index=size - (int(testpercentage * size))\n",
    "    \n",
    "    train_lines=lines[:index]\n",
    "    #print index\n",
    "    test_lines=lines[index:]\n",
    "\n",
    "    train_labels=  [l.split()[0] for l in train_lines]\n",
    "\n",
    "\n",
    "    train_documents=  [l.split()[1:] for l in train_lines]\n",
    "    \n",
    "    test_labels=  [l.split()[0] for l in test_lines]\n",
    "\n",
    "\n",
    "    test_documents=  [l.split()[1:] for l in test_lines]\n",
    "    \n",
    "    return train_labels, train_documents, test_labels, test_documents\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20000\n",
      "5000\n",
      "5000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(396, 198)"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels, train_documents, test_labels, test_documents = getTrainDocsSplit(0.20)\n",
    "\n",
    "#train_documents = filterLen(train_documents, 2)\n",
    "#test_documents =filterLen(test_documents, 2)\n",
    "\n",
    "#test_documents =getTestDocs()\n",
    "#test_documents =filterLen(test_documents, 2)\n",
    "\n",
    "print len(train_documents)\n",
    "print len(test_documents)\n",
    "print len(test_labels)\n",
    "\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "count_vect = CountVectorizer()\n",
    "X_train_counts = count_vect.fit_transform(train_documents[0])\n",
    "X_train_counts.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "import string\n",
    "import collections\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def cmer(name, c=3):\n",
    "    r\"\"\" Given a name and parameter c, return the vector of c-mers associated with the name\n",
    "    \"\"\"\n",
    "    name = name.lower()\n",
    "    \n",
    "    ### FILL IN THE BLANKS ###\n",
    "    \n",
    "    v = []\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    return v\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def build_matrix(docs):\n",
    "    r\"\"\" Build sparse matrix from a list of documents, \n",
    "    each of which is a list of word/terms in the document.  \n",
    "    \"\"\"\n",
    "    nrows = len(docs)\n",
    "    idx = {}\n",
    "    tid = 0\n",
    "    nnz = 0\n",
    "    for d in docs:\n",
    "        nnz += len(set(d))\n",
    "        for w in d:\n",
    "            if w not in idx:\n",
    "                idx[w] = tid\n",
    "                tid += 1\n",
    "    ncols = len(idx)\n",
    "        \n",
    "    # set up memory\n",
    "    ind = np.zeros(nnz, dtype=np.int)\n",
    "    val = np.zeros(nnz, dtype=np.double)\n",
    "    ptr = np.zeros(nrows+1, dtype=np.int)\n",
    "    i = 0  # document ID / row counter\n",
    "    n = 0  # non-zero counter\n",
    "    # transfer values\n",
    "    for d in docs:\n",
    "        cnt = Counter(d)\n",
    "        #print cnt\n",
    "        keys = list(k for k,_ in cnt.most_common())\n",
    "        #print keys\n",
    "        l = len(keys)\n",
    "        for j,k in enumerate(keys):\n",
    "            ind[j+n] = idx[k]\n",
    "            val[j+n] = cnt[k]\n",
    "        ptr[i+1] = ptr[i] + l\n",
    "        n += l\n",
    "        i += 1\n",
    "            \n",
    "    mat = csr_matrix((val, ind, ptr), shape=(nrows, ncols), dtype=np.double)\n",
    "    mat.sort_indices()\n",
    "    \n",
    "    return mat,idx\n",
    "\n",
    "\n",
    "def build_test_matrix(docs,idx):\n",
    "    r\"\"\" Build sparse matrix from a list of documents, \n",
    "    each of which is a list of word/terms in the document.  \n",
    "    \"\"\"\n",
    "    nrows = len(docs)\n",
    "    \n",
    "    \n",
    "    nnz = 0\n",
    "    test_docs = []\n",
    "    for d in docs:\n",
    "        copy_d=[]\n",
    "     \n",
    "        for w in d:\n",
    "            if w in idx:\n",
    "                copy_d.append(w)\n",
    "        \n",
    "        nnz += len(set(copy_d))\n",
    "        test_docs.append(copy_d)\n",
    "        \n",
    "        \n",
    "    ncols = len(idx)\n",
    "        \n",
    "    # set up memory\n",
    "    ind = np.zeros(nnz, dtype=np.int)\n",
    "    val = np.zeros(nnz, dtype=np.double)\n",
    "    ptr = np.zeros(nrows+1, dtype=np.int)\n",
    "    i = 0  # document ID / row counter\n",
    "    n = 0  # non-zero counter\n",
    "    # transfer values\n",
    "    for d in test_docs:\n",
    "        cnt = Counter(d)\n",
    "        #print cnt\n",
    "        keys = list(k for k,_ in cnt.most_common())\n",
    "        #print keys\n",
    "        l = len(keys)\n",
    "        for j,k in enumerate(keys):\n",
    "            ind[j+n] = idx[k]\n",
    "            val[j+n] = cnt[k]\n",
    "        ptr[i+1] = ptr[i] + l\n",
    "        n += l\n",
    "        i += 1\n",
    "            \n",
    "    mat = csr_matrix((val, ind, ptr), shape=(nrows, ncols), dtype=np.double)\n",
    "    mat.sort_indices()\n",
    "    \n",
    "    return mat\n",
    "\n",
    "def csr_info(mat, name=\"\", non_empy=False):\n",
    "    r\"\"\" Print out info about this CSR matrix. If non_empy, \n",
    "    report number of non-empty rows and cols as well\n",
    "    \"\"\"\n",
    "    if non_empy:\n",
    "        print(\"%s [nrows %d (%d non-empty), ncols %d (%d non-empty), nnz %d]\" % (\n",
    "                name, mat.shape[0], \n",
    "                sum(1 if mat.indptr[i+1] > mat.indptr[i] else 0 \n",
    "                for i in range(mat.shape[0])), \n",
    "                mat.shape[1], len(np.unique(mat.indices)), \n",
    "                len(mat.data)))\n",
    "    else:\n",
    "        print( \"%s [nrows %d, ncols %d, nnz %d]\" % (name, \n",
    "                mat.shape[0], mat.shape[1], len(mat.data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [nrows 20000, ncols 245284, nnz 3018850]\n",
      " [nrows 5000, ncols 245284, nnz 725685]\n"
     ]
    }
   ],
   "source": [
    "mat,word_dict = build_matrix(train_documents)\n",
    "\n",
    "\n",
    "\n",
    "test_mat =build_test_matrix(test_documents,word_dict)\n",
    "csr_info(mat)\n",
    "csr_info(test_mat)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('mat1:', matrix([[  2.,  11.,   1.,   1.,   1.,   1.,   6.,   1.,   1.,   1.,   5.,\n",
      "           1.]]), '\\n')\n",
      "('mat2:', matrix([[ 7.22383683,  0.45076082,  0.76346235,  0.39060113,  4.85363155,\n",
      "          6.14228744,  0.70999912,  2.35651344,  2.55040563,  5.43757943,\n",
      "          3.78469657,  1.91223362]]), '\\n')\n",
      "('mat3:', matrix([[ 0.08619249,  0.00537833,  0.00910939,  0.00466053,  0.05791196,\n",
      "          0.07328779,  0.00847148,  0.02811716,  0.03043062,  0.06487944,\n",
      "          0.04515778,  0.02281615]]))\n"
     ]
    }
   ],
   "source": [
    "# scale matrix and normalize its rows\n",
    "def csr_idf(mat, copy=False, **kargs):\n",
    "    r\"\"\" Scale a CSR matrix by idf. \n",
    "    Returns scaling factors as dict. If copy is True, \n",
    "    returns scaled matrix and scaling factors.\n",
    "    \"\"\"\n",
    "    if copy is True:\n",
    "        mat = mat.copy()\n",
    "    nrows = mat.shape[0]\n",
    "    nnz = mat.nnz\n",
    "    ind, val, ptr = mat.indices, mat.data, mat.indptr\n",
    "    # document frequency\n",
    "    df = defaultdict(int)\n",
    "    for i in ind:\n",
    "        df[i] += 1\n",
    "    # inverse document frequency\n",
    "    for k,v in df.items():\n",
    "        df[k] = np.log(nrows / float(v))  ## df turns to idf - reusing memory\n",
    "    # scale by idf\n",
    "    for i in range(0, nnz):\n",
    "        val[i] *= df[ind[i]]\n",
    "        \n",
    "    return df if copy is False else mat\n",
    "\n",
    "def csr_l2normalize(mat, copy=False, **kargs):\n",
    "    r\"\"\" Normalize the rows of a CSR matrix by their L-2 norm. \n",
    "    If copy is True, returns a copy of the normalized matrix.\n",
    "    \"\"\"\n",
    "    if copy is True:\n",
    "        mat = mat.copy()\n",
    "    nrows = mat.shape[0]\n",
    "    nnz = mat.nnz\n",
    "    ind, val, ptr = mat.indices, mat.data, mat.indptr\n",
    "    # normalize\n",
    "    for i in range(nrows):\n",
    "        rsum = 0.0    \n",
    "        for j in range(ptr[i], ptr[i+1]):\n",
    "            rsum += val[j]**2\n",
    "        if rsum == 0.0:\n",
    "            continue  # do not normalize empty rows\n",
    "        rsum = 1.0/np.sqrt(rsum)\n",
    "        for j in range(ptr[i], ptr[i+1]):\n",
    "            val[j] *= rsum\n",
    "            \n",
    "    if copy is True:\n",
    "        return mat\n",
    "mat2 = csr_idf(mat, copy=True)\n",
    "mat3 = csr_l2normalize(mat2, copy=True)\n",
    "\n",
    "test_mat2 = csr_idf(test_mat, copy=True)\n",
    "test_mat3 = csr_l2normalize(test_mat2, copy=True)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print(\"mat1:\", mat[0,:12].todense(), \"\\n\")\n",
    "print(\"mat2:\", mat2[0,:12].todense(), \"\\n\")\n",
    "print(\"mat3:\", mat3[0,:12].todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "from lsh import clsh, jlsh, generateSamples, findNeighborsBrute, recall\n",
    "from scipy.sparse.linalg import norm\n",
    "\n",
    "def splitData(mat, cls, fold=1, d=10):\n",
    "    r\"\"\" Split the matrix and class info into train and test data using d-fold hold-out\n",
    "    \"\"\"\n",
    "    n = mat.shape[0]\n",
    "    r = int(np.ceil(n*1.0/d))\n",
    "    mattr = []\n",
    "    clstr = []\n",
    "    # split mat and cls into d folds\n",
    "    for f in range(d):\n",
    "        if f+1 != fold:\n",
    "            mattr.append( mat[f*r: min((f+1)*r, n)] )\n",
    "            clstr.extend( cls[f*r: min((f+1)*r, n)] )\n",
    "    # join all fold matrices that are not the test matrix\n",
    "    train = sp.vstack(mattr, format='csr')\n",
    "    # extract the test matrix and class values associated with the test rows\n",
    "    test = mat[(fold-1)*r: min(fold*r, n), :]\n",
    "    clste = cls[(fold-1)*r: min(fold*r, n)]\n",
    "\n",
    "    return train, clstr, test, clste\n",
    "\n",
    "\n",
    "def cosineSimilarity(vector1, vector2):\n",
    "    distance = 0\n",
    "    return vector1.dot(vector2.T).todense().item() / ( norm(vector1) * norm(vector2) )\n",
    "\n",
    "\n",
    "import operator \n",
    "\n",
    "def getKey(item):\n",
    "    return item[1]\n",
    "\n",
    "def getNeighbors(trainingSet, testInstance, k):\n",
    "    distances = []\n",
    "    distances=testInstance.dot(trainingSet.T)\n",
    "    sims = list(zip(distances.indices, distances.data))\n",
    "    sims.sort(key=lambda x: x[1], reverse=True)\n",
    "    \n",
    "    poscount=0\n",
    "    negcount=0\n",
    "    neighbors = [s[0] for s in sims[:k] if s[1] > 0 ]\n",
    "    \n",
    "    for x in range(k):\n",
    "       \n",
    "        if(train_labels[neighbors[x]]=='-1'):\n",
    "            negcount+=1  \n",
    "        else:\n",
    "            poscount+=1\n",
    "    \n",
    "    label='+1'\n",
    "    if(negcount>poscount):\n",
    "        label='-1'\n",
    "\n",
    "    return label\n",
    "\n",
    "def getNeighborsWhole(trainingSet, testSet, k):\n",
    "    distances = []\n",
    "    num_rows, num_cols = testSet.shape\n",
    "    labels=[]\n",
    "    distances=testSet.dot(trainingSet.T).todense()\n",
    "    \n",
    "    \n",
    "\n",
    "    distances =np.array(distances)\n",
    "    indixes= np.argsort(distances,axis=1)[::-1]\n",
    "    indixes=np.array(indixes)\n",
    "\n",
    "    print indixes.shape\n",
    "\n",
    "    \n",
    "    for row in range(num_rows):\n",
    "        poscount=0\n",
    "        negcount=0\n",
    "        l=indixes[row]\n",
    "        d=distances[row]\n",
    "        for x in range(k):\n",
    "            dist=d[x]\n",
    "            if(dist>0):\n",
    "                if(train_labels[l[x]]=='-1'):\n",
    "                    negcount+=1  \n",
    "                else:\n",
    "                    poscount+=1\n",
    "        label='+1'\n",
    "        if(negcount>poscount):\n",
    "            label='-1'\n",
    "        elif(negcount==poscount):\n",
    "            label=train_labels[l[0]]\n",
    "            \n",
    "        labels.append(label)\n",
    "        \n",
    "        \n",
    "    \n",
    "\n",
    "    return labels\n",
    "\n",
    "def calc_accuracy(pred,test):\n",
    "    correct=0\n",
    "    \n",
    "    for i in range(len(pred)):\n",
    "        if(pred[i]==test[i]):\n",
    "            correct+=1;\n",
    "    \n",
    "    print correct\n",
    "    perc=(float)(correct/len(pred))\n",
    "    \n",
    "    return perc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5000\n",
      "+1\n",
      "+1\n",
      "+1\n",
      "+1\n",
      "+1\n",
      "5000\n",
      "3620\n",
      "0.0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import os\n",
    "\n",
    "filename = 'output.dat'\n",
    "\n",
    "highscore = open(filename,'w')\n",
    "\n",
    "\n",
    "num_rows, num_cols = test_mat3.shape\n",
    "print num_rows\n",
    "\n",
    "\n",
    "\n",
    "#labels = getNeighborsWhole(mat3,test_mat3,5)\n",
    "labels=[]\n",
    "\n",
    "i=0\n",
    "for i in range(num_rows):\n",
    "    l =getNeighbors(mat3,test_mat3[i],5)\n",
    "    if(i%1000==0):\n",
    "        print l   \n",
    "    labels.append(l)\n",
    "\n",
    "print len(labels)\n",
    "ac= calc_accuracy(labels,test_labels)\n",
    "print ac\n",
    "\n",
    "highscore.close()  \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def l2Ap(trainingSet, epsilon=0.5):\n",
    "    \n",
    "    num_rows, num_cols = trainingSet.shape\n",
    "    l2ap_index=[]\n",
    "    \n",
    "    for i in range(num_rows):\n",
    "        p,l2normprefix=findP(num_rows[i],epsilon)\n",
    "        l2ap_index.append(l2normprefix, p,)\n",
    "        \n",
    "        \n",
    "        \n",
    "    \n",
    "return 1 \n",
    "\n",
    "\n",
    "def findP(document,epsilon=0.5):\n",
    "     features = len(document)\n",
    "     l2normprefix=0\n",
    "     for f in range(features):\n",
    "         l2normprefix+=features[f]*features[f]\n",
    "         nrm= math.sqrt(l2normprefix)\n",
    "         if(nrm>epsilon):\n",
    "            l2normprefix=nrm\n",
    "            break\n",
    "\n",
    "return f-1,l2normprefix\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[100  30   3  90]\n",
      " [  2   2   7  92]\n",
      " [  2   2   7  92]]\n",
      "[[0 1 2 3]\n",
      " [0 1 2 3]\n",
      " [2 1 3 0]]\n",
      "0 0 0\n",
      "0 1 1\n",
      "0 2 2\n",
      "0 3 3\n",
      "1 0 0\n",
      "1 1 1\n",
      "1 2 2\n",
      "1 3 3\n",
      "2 0 2\n",
      "2 1 1\n",
      "2 2 3\n",
      "2 3 0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "x = np.array([[100, 30,3,90], [2, 2,7,92],[2, 2,7,92]])\n",
    "print x\n",
    "\n",
    "x = np.argsort(x, axis=1)[::-1]\n",
    "\n",
    "print x\n",
    "\n",
    "for (i,row) in enumerate(x):\n",
    "        for (j,value) in enumerate(row):\n",
    "            print i,j,value\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "ename": "LookupError",
     "evalue": "\n**********************************************************************\n  Resource u'corpora/stopwords' not found.  Please use the NLTK\n  Downloader to obtain the resource:  >>> nltk.download()\n  Searched in:\n    - '/Users/rashmisharma/nltk_data'\n    - '/usr/share/nltk_data'\n    - '/usr/local/share/nltk_data'\n    - '/usr/lib/nltk_data'\n    - '/usr/local/lib/nltk_data'\n**********************************************************************",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-196-aaa07745fba4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\"\"\" Transform texts to Tf-Idf coordinates and cluster texts using K-Means \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m vectorizer = TfidfVectorizer(tokenizer=process_text,\n\u001b[0;32m---> 29\u001b[0;31m                                  \u001b[0mstop_words\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstopwords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'english'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m                                  \u001b[0mmax_df\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.95\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m                                  \u001b[0mmin_df\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/rashmisharma/anaconda2/lib/python2.7/site-packages/nltk/corpus/util.pyc\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, attr)\u001b[0m\n\u001b[1;32m    114\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"LazyCorpusLoader object has no attribute '__bases__'\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 116\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__load\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    117\u001b[0m         \u001b[0;31m# This looks circular, but its not, since __load() changes our\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m         \u001b[0;31m# __class__ to something new:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/rashmisharma/anaconda2/lib/python2.7/site-packages/nltk/corpus/util.pyc\u001b[0m in \u001b[0;36m__load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mroot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{}/{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzip_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m                 \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m         \u001b[0;31m# Load the corpus.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource u'corpora/stopwords' not found.  Please use the NLTK\n  Downloader to obtain the resource:  >>> nltk.download()\n  Searched in:\n    - '/Users/rashmisharma/nltk_data'\n    - '/usr/share/nltk_data'\n    - '/usr/local/share/nltk_data'\n    - '/usr/lib/nltk_data'\n    - '/usr/local/lib/nltk_data'\n**********************************************************************"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from pprint import pprint\n",
    "import string\n",
    "import collections\n",
    "\n",
    "def process_text(text, stem=True):\n",
    "    \"\"\" Tokenize text and stem words removing punctuation \"\"\"\n",
    "    text = text.translate(string.punctuation)\n",
    "\n",
    "\n",
    "    tokens = word_tokenize(text)\n",
    "    if len(tokens)!=0:\n",
    "        if stem:\n",
    "            stemmer = PorterStemmer()\n",
    "            tokens = [stemmer.stem(t.lower()) for t in tokens]\n",
    "    else:\n",
    "        tokens=[]\n",
    "        tokens.append(text)\n",
    "\n",
    "    return tokens\n",
    "\n",
    "\n",
    "\"\"\" Transform texts to Tf-Idf coordinates and cluster texts using K-Means \"\"\"\n",
    "vectorizer = TfidfVectorizer(tokenizer=process_text,\n",
    "                                 stop_words=stopwords.words('english'),\n",
    "                                 max_df=0.95,\n",
    "                                 min_df=0.0,\n",
    "                                 lowercase=True)\n",
    "\n",
    "text=\"This movie was horrible. I swear they didn't even write a script they just kinda winged it through out the whole movie. Ice-T was annoying as hell. *SPOILERS Phht more like reasons not to watch it* They sit down and eat breakfast for 20 minutes. he coulda been long gone. The ground was hard it would of been close to impossible to to track him with out dogs. And when ICE-T is on that Hill and uses that Spaz-15 Assault SHOTGUN\"\n",
    "\n",
    "tfidf_model = vectorizer.fit_transform(text)\n",
    "\n",
    "print tfidf_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}