Avg W2V with 200-D.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading Avg w2v vectors(200-D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 201)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>191</th>\n",
       "      <th>192</th>\n",
       "      <th>193</th>\n",
       "      <th>194</th>\n",
       "      <th>195</th>\n",
       "      <th>196</th>\n",
       "      <th>197</th>\n",
       "      <th>198</th>\n",
       "      <th>199</th>\n",
       "      <th>200</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000389</td>\n",
       "      <td>-0.000173</td>\n",
       "      <td>0.000436</td>\n",
       "      <td>-0.000163</td>\n",
       "      <td>0.000199</td>\n",
       "      <td>0.000180</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>0.000549</td>\n",
       "      <td>0.000178</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000026</td>\n",
       "      <td>-0.000466</td>\n",
       "      <td>-0.000134</td>\n",
       "      <td>0.000429</td>\n",
       "      <td>-0.000480</td>\n",
       "      <td>0.000480</td>\n",
       "      <td>-0.000280</td>\n",
       "      <td>-0.000104</td>\n",
       "      <td>0.000497</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000284</td>\n",
       "      <td>0.000089</td>\n",
       "      <td>0.000249</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>-0.000130</td>\n",
       "      <td>0.000052</td>\n",
       "      <td>-0.000071</td>\n",
       "      <td>0.000123</td>\n",
       "      <td>-0.000138</td>\n",
       "      <td>-0.000318</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000117</td>\n",
       "      <td>-0.000317</td>\n",
       "      <td>-0.000018</td>\n",
       "      <td>-0.000481</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>-0.000097</td>\n",
       "      <td>0.000324</td>\n",
       "      <td>-0.000045</td>\n",
       "      <td>-0.000154</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.000061</td>\n",
       "      <td>-0.000244</td>\n",
       "      <td>0.000047</td>\n",
       "      <td>-0.000635</td>\n",
       "      <td>-0.000620</td>\n",
       "      <td>0.000108</td>\n",
       "      <td>0.000711</td>\n",
       "      <td>-0.000231</td>\n",
       "      <td>-0.000256</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000208</td>\n",
       "      <td>-0.000340</td>\n",
       "      <td>-0.000010</td>\n",
       "      <td>0.000276</td>\n",
       "      <td>0.000060</td>\n",
       "      <td>0.000266</td>\n",
       "      <td>0.000313</td>\n",
       "      <td>-0.000156</td>\n",
       "      <td>0.000381</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000390</td>\n",
       "      <td>-0.000164</td>\n",
       "      <td>0.000224</td>\n",
       "      <td>-0.000373</td>\n",
       "      <td>-0.000065</td>\n",
       "      <td>-0.000066</td>\n",
       "      <td>0.000333</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>-0.000021</td>\n",
       "      <td>0.000121</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000292</td>\n",
       "      <td>-0.000301</td>\n",
       "      <td>-0.000081</td>\n",
       "      <td>-0.000084</td>\n",
       "      <td>-0.000054</td>\n",
       "      <td>-0.000090</td>\n",
       "      <td>-0.000026</td>\n",
       "      <td>0.000211</td>\n",
       "      <td>-0.000205</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000057</td>\n",
       "      <td>0.000510</td>\n",
       "      <td>-0.000419</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>-0.000151</td>\n",
       "      <td>-0.000305</td>\n",
       "      <td>-0.000428</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000173</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000019</td>\n",
       "      <td>0.000849</td>\n",
       "      <td>-0.000127</td>\n",
       "      <td>0.000260</td>\n",
       "      <td>-0.000177</td>\n",
       "      <td>0.000521</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>-0.000096</td>\n",
       "      <td>0.000205</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 201 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0         1         2         3         4         5         6    \\\n",
       "0  0.000389 -0.000173  0.000436 -0.000163  0.000199  0.000180  0.000041   \n",
       "1  0.000284  0.000089  0.000249  0.000084 -0.000130  0.000052 -0.000071   \n",
       "2 -0.000061 -0.000244  0.000047 -0.000635 -0.000620  0.000108  0.000711   \n",
       "3  0.000390 -0.000164  0.000224 -0.000373 -0.000065 -0.000066  0.000333   \n",
       "4  0.000057  0.000510 -0.000419  0.000030 -0.000151 -0.000305 -0.000428   \n",
       "\n",
       "        7         8         9   ...        191       192       193       194  \\\n",
       "0  0.000549  0.000178  0.000004 ...  -0.000026 -0.000466 -0.000134  0.000429   \n",
       "1  0.000123 -0.000138 -0.000318 ...  -0.000117 -0.000317 -0.000018 -0.000481   \n",
       "2 -0.000231 -0.000256  0.000566 ...   0.000208 -0.000340 -0.000010  0.000276   \n",
       "3 -0.000012 -0.000021  0.000121 ...   0.000292 -0.000301 -0.000081 -0.000084   \n",
       "4  0.000091  0.000022  0.000173 ...  -0.000019  0.000849 -0.000127  0.000260   \n",
       "\n",
       "        195       196       197       198       199  200  \n",
       "0 -0.000480  0.000480 -0.000280 -0.000104  0.000497    1  \n",
       "1  0.000017 -0.000097  0.000324 -0.000045 -0.000154    1  \n",
       "2  0.000060  0.000266  0.000313 -0.000156  0.000381    0  \n",
       "3 -0.000054 -0.000090 -0.000026  0.000211 -0.000205    1  \n",
       "4 -0.000177  0.000521  0.000027 -0.000096  0.000205    1  \n",
       "\n",
       "[5 rows x 201 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_w2v =pd.read_pickle(\"avg_w2v_vec_200\")\n",
    "print(avg_w2v.shape)\n",
    "avg_w2v.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x =avg_w2v.iloc[:,:200]\n",
    "y =avg_w2v.iloc[:,200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 200)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(122109,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x.shape)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Column Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a =StandardScaler()\n",
    "x =a.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test =train_test_split(x,y,test_size =0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(85476, 200)\n",
      "(36633, 200)\n",
      "(85476,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(36633,)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x_train.shape)\n",
    "print(x_test.shape)\n",
    "print(y_train.shape)\n",
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tuned_params = [{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid=[{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = GridSearchCV(LogisticRegression(), tuned_params, scoring = 'accuracy')\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression(C=0.01, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False)\n",
      "0.756612890017\n"
     ]
    }
   ],
   "source": [
    "print(model.best_estimator_)\n",
    "print(model.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.best_estimator_.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75.661289001719751"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_pred, normalize=True) * float(100)\n",
    "acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can clearly see increasing dimenions the model performs better than 100-D as the chances of data becomes linearly separable in higher dimensions increases.So we get the best hyperplane as dimensions increases due to which model behaves better."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[12493,  4193],\n",
       "       [ 4723, 15224]], dtype=int64)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## L1 regularization and sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_    \n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This means 200 out of 200 are non zero.That means for regularization takes place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "198\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see clearly as c decreases sparsity increases.This time 198 features out of 200  are non zero,means 2 features are removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.001, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the 94 out of 200 are non zero.That means 106 are zero which are removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.00025, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At c=0.00025 we are getting top 15 features and rest becomes zero."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}