Avg W2V with 100-D.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading Avg w2v vectors(100-D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 101)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "      <th>100</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000778</td>\n",
       "      <td>-0.000347</td>\n",
       "      <td>0.000871</td>\n",
       "      <td>-0.000326</td>\n",
       "      <td>0.000397</td>\n",
       "      <td>0.000361</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>0.001098</td>\n",
       "      <td>0.000355</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000193</td>\n",
       "      <td>0.000537</td>\n",
       "      <td>0.000594</td>\n",
       "      <td>0.000397</td>\n",
       "      <td>-0.000584</td>\n",
       "      <td>-0.001169</td>\n",
       "      <td>0.000315</td>\n",
       "      <td>-0.000140</td>\n",
       "      <td>0.000049</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000567</td>\n",
       "      <td>0.000178</td>\n",
       "      <td>0.000499</td>\n",
       "      <td>0.000169</td>\n",
       "      <td>-0.000260</td>\n",
       "      <td>0.000105</td>\n",
       "      <td>-0.000142</td>\n",
       "      <td>0.000246</td>\n",
       "      <td>-0.000276</td>\n",
       "      <td>-0.000637</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000212</td>\n",
       "      <td>0.000260</td>\n",
       "      <td>0.000580</td>\n",
       "      <td>0.000210</td>\n",
       "      <td>0.000176</td>\n",
       "      <td>0.000737</td>\n",
       "      <td>0.000288</td>\n",
       "      <td>0.000127</td>\n",
       "      <td>0.000436</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.000123</td>\n",
       "      <td>-0.000488</td>\n",
       "      <td>0.000094</td>\n",
       "      <td>-0.001271</td>\n",
       "      <td>-0.001240</td>\n",
       "      <td>0.000217</td>\n",
       "      <td>0.001422</td>\n",
       "      <td>-0.000463</td>\n",
       "      <td>-0.000512</td>\n",
       "      <td>0.001132</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000842</td>\n",
       "      <td>0.000572</td>\n",
       "      <td>0.000398</td>\n",
       "      <td>0.000072</td>\n",
       "      <td>0.000744</td>\n",
       "      <td>0.000469</td>\n",
       "      <td>-0.000092</td>\n",
       "      <td>-0.000362</td>\n",
       "      <td>-0.000065</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000780</td>\n",
       "      <td>-0.000327</td>\n",
       "      <td>0.000449</td>\n",
       "      <td>-0.000747</td>\n",
       "      <td>-0.000130</td>\n",
       "      <td>-0.000132</td>\n",
       "      <td>0.000667</td>\n",
       "      <td>-0.000025</td>\n",
       "      <td>-0.000042</td>\n",
       "      <td>0.000242</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000111</td>\n",
       "      <td>0.000417</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>-0.000190</td>\n",
       "      <td>-0.000268</td>\n",
       "      <td>0.000645</td>\n",
       "      <td>0.000049</td>\n",
       "      <td>-0.000225</td>\n",
       "      <td>-0.000452</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000114</td>\n",
       "      <td>0.001019</td>\n",
       "      <td>-0.000837</td>\n",
       "      <td>0.000060</td>\n",
       "      <td>-0.000303</td>\n",
       "      <td>-0.000609</td>\n",
       "      <td>-0.000855</td>\n",
       "      <td>0.000182</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>0.000346</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000147</td>\n",
       "      <td>-0.000636</td>\n",
       "      <td>0.000251</td>\n",
       "      <td>0.001050</td>\n",
       "      <td>0.000173</td>\n",
       "      <td>-0.000852</td>\n",
       "      <td>-0.000106</td>\n",
       "      <td>0.000595</td>\n",
       "      <td>0.000065</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 101 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0         1         2         3         4         5         6    \\\n",
       "0  0.000778 -0.000347  0.000871 -0.000326  0.000397  0.000361  0.000081   \n",
       "1  0.000567  0.000178  0.000499  0.000169 -0.000260  0.000105 -0.000142   \n",
       "2 -0.000123 -0.000488  0.000094 -0.001271 -0.001240  0.000217  0.001422   \n",
       "3  0.000780 -0.000327  0.000449 -0.000747 -0.000130 -0.000132  0.000667   \n",
       "4  0.000114  0.001019 -0.000837  0.000060 -0.000303 -0.000609 -0.000855   \n",
       "\n",
       "        7         8         9   ...        91        92        93        94   \\\n",
       "0  0.001098  0.000355  0.000008 ...  -0.000193  0.000537  0.000594  0.000397   \n",
       "1  0.000246 -0.000276 -0.000637 ...  -0.000212  0.000260  0.000580  0.000210   \n",
       "2 -0.000463 -0.000512  0.001132 ...   0.000842  0.000572  0.000398  0.000072   \n",
       "3 -0.000025 -0.000042  0.000242 ...  -0.000111  0.000417  0.000003 -0.000190   \n",
       "4  0.000182  0.000044  0.000346 ...  -0.000147 -0.000636  0.000251  0.001050   \n",
       "\n",
       "        95        96        97        98        99   100  \n",
       "0 -0.000584 -0.001169  0.000315 -0.000140  0.000049    1  \n",
       "1  0.000176  0.000737  0.000288  0.000127  0.000436    1  \n",
       "2  0.000744  0.000469 -0.000092 -0.000362 -0.000065    0  \n",
       "3 -0.000268  0.000645  0.000049 -0.000225 -0.000452    1  \n",
       "4  0.000173 -0.000852 -0.000106  0.000595  0.000065    1  \n",
       "\n",
       "[5 rows x 101 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_w2v =pd.read_pickle(\"avg_w2v_vec_100\")\n",
    "print(avg_w2v.shape)\n",
    "avg_w2v.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "x =avg_w2v.iloc[:,:100]\n",
    "y =avg_w2v.iloc[:,100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 100)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(122109,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x.shape)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Column Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a =StandardScaler()\n",
    "x =a.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test =train_test_split(x,y,test_size =0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(85476, 100)\n",
      "(36633, 100)\n",
      "(85476,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(36633,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x_train.shape)\n",
    "print(x_test.shape)\n",
    "print(y_train.shape)\n",
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tuned_params = [{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid=[{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = GridSearchCV(LogisticRegression(), tuned_params, scoring = 'accuracy')\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression(C=1, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False)\n",
      "0.710889089073\n"
     ]
    }
   ],
   "source": [
    "print(model.best_estimator_)\n",
    "print(model.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.best_estimator_.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "71.088908907269399"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_pred, normalize=True) * float(100)\n",
    "acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11548,  4896],\n",
       "       [ 5695, 14494]], dtype=int64)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L1 regularization and sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_    \n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This means 100 out of 100 are non zero.That means for regularization takes place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see clearly as c decreases sparsity increases.This time 99 features out of 100  are non zero,means 1 features is removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.001, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the 55 out of 100 are non zero.That means 45 are zero which are removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.00025, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At c=0.00025 we are getting top 10 features and rest becomes zero."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}