TFIDF Avg W2V with100-D.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading TFIDF Avg w2v vectors(100-D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 101)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "      <th>100</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.001005</td>\n",
       "      <td>-0.000993</td>\n",
       "      <td>-0.000176</td>\n",
       "      <td>-0.000016</td>\n",
       "      <td>-0.000637</td>\n",
       "      <td>0.001346</td>\n",
       "      <td>0.000667</td>\n",
       "      <td>0.001510</td>\n",
       "      <td>-0.000133</td>\n",
       "      <td>-0.000779</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001775</td>\n",
       "      <td>-0.000025</td>\n",
       "      <td>-0.001377</td>\n",
       "      <td>0.000550</td>\n",
       "      <td>-0.000228</td>\n",
       "      <td>-0.001434</td>\n",
       "      <td>-0.000971</td>\n",
       "      <td>0.000378</td>\n",
       "      <td>-0.001934</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000447</td>\n",
       "      <td>0.000047</td>\n",
       "      <td>-0.000051</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000638</td>\n",
       "      <td>0.000663</td>\n",
       "      <td>-0.001048</td>\n",
       "      <td>-0.000657</td>\n",
       "      <td>0.000902</td>\n",
       "      <td>-0.000835</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000874</td>\n",
       "      <td>-0.000076</td>\n",
       "      <td>-0.000138</td>\n",
       "      <td>0.000227</td>\n",
       "      <td>0.000570</td>\n",
       "      <td>-0.000014</td>\n",
       "      <td>-0.000653</td>\n",
       "      <td>0.000828</td>\n",
       "      <td>-0.000387</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.002199</td>\n",
       "      <td>0.000842</td>\n",
       "      <td>-0.001726</td>\n",
       "      <td>-0.000084</td>\n",
       "      <td>-0.000796</td>\n",
       "      <td>0.000935</td>\n",
       "      <td>0.000353</td>\n",
       "      <td>0.000058</td>\n",
       "      <td>0.001372</td>\n",
       "      <td>-0.000609</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000939</td>\n",
       "      <td>0.000222</td>\n",
       "      <td>0.000223</td>\n",
       "      <td>-0.000879</td>\n",
       "      <td>-0.001259</td>\n",
       "      <td>-0.001296</td>\n",
       "      <td>0.000505</td>\n",
       "      <td>-0.000455</td>\n",
       "      <td>0.001263</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000479</td>\n",
       "      <td>-0.000181</td>\n",
       "      <td>0.000898</td>\n",
       "      <td>0.000729</td>\n",
       "      <td>-0.001107</td>\n",
       "      <td>0.000479</td>\n",
       "      <td>-0.000218</td>\n",
       "      <td>-0.000634</td>\n",
       "      <td>-0.000136</td>\n",
       "      <td>-0.000369</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000357</td>\n",
       "      <td>0.000199</td>\n",
       "      <td>0.000379</td>\n",
       "      <td>0.001250</td>\n",
       "      <td>-0.000416</td>\n",
       "      <td>0.000148</td>\n",
       "      <td>0.000612</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>0.000545</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.001807</td>\n",
       "      <td>0.000264</td>\n",
       "      <td>-0.001362</td>\n",
       "      <td>-0.001132</td>\n",
       "      <td>-0.001522</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.001637</td>\n",
       "      <td>-0.000874</td>\n",
       "      <td>0.000608</td>\n",
       "      <td>0.000235</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001261</td>\n",
       "      <td>0.000399</td>\n",
       "      <td>-0.000921</td>\n",
       "      <td>0.001763</td>\n",
       "      <td>-0.000563</td>\n",
       "      <td>-0.001746</td>\n",
       "      <td>0.001239</td>\n",
       "      <td>0.001081</td>\n",
       "      <td>0.000124</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 101 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0         1         2         3         4         5         6    \\\n",
       "0  0.001005 -0.000993 -0.000176 -0.000016 -0.000637  0.001346  0.000667   \n",
       "1  0.000447  0.000047 -0.000051  0.000002  0.000638  0.000663 -0.001048   \n",
       "2  0.002199  0.000842 -0.001726 -0.000084 -0.000796  0.000935  0.000353   \n",
       "3  0.000479 -0.000181  0.000898  0.000729 -0.001107  0.000479 -0.000218   \n",
       "4  0.001807  0.000264 -0.001362 -0.001132 -0.001522  0.000012  0.001637   \n",
       "\n",
       "        7         8         9   ...        91        92        93        94   \\\n",
       "0  0.001510 -0.000133 -0.000779 ...   0.001775 -0.000025 -0.001377  0.000550   \n",
       "1 -0.000657  0.000902 -0.000835 ...  -0.000874 -0.000076 -0.000138  0.000227   \n",
       "2  0.000058  0.001372 -0.000609 ...  -0.000939  0.000222  0.000223 -0.000879   \n",
       "3 -0.000634 -0.000136 -0.000369 ...  -0.000357  0.000199  0.000379  0.001250   \n",
       "4 -0.000874  0.000608  0.000235 ...   0.001261  0.000399 -0.000921  0.001763   \n",
       "\n",
       "        95        96        97        98        99   100  \n",
       "0 -0.000228 -0.001434 -0.000971  0.000378 -0.001934    1  \n",
       "1  0.000570 -0.000014 -0.000653  0.000828 -0.000387    1  \n",
       "2 -0.001259 -0.001296  0.000505 -0.000455  0.001263    0  \n",
       "3 -0.000416  0.000148  0.000612 -0.000012  0.000545    1  \n",
       "4 -0.000563 -0.001746  0.001239  0.001081  0.000124    1  \n",
       "\n",
       "[5 rows x 101 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_avg_w2v =pd.read_pickle(\"tfidf_avg_vec_100\")\n",
    "print(tfidf_avg_w2v.shape)\n",
    "tfidf_avg_w2v.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x =tfidf_avg_w2v.iloc[:,:100]\n",
    "y =tfidf_avg_w2v.iloc[:,100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 100)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(122109,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x.shape)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Column Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a =StandardScaler()\n",
    "x =a.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test =train_test_split(x,y,test_size =0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(85476, 100)\n",
      "(36633, 100)\n",
      "(85476,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(36633,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x_train.shape)\n",
    "print(x_test.shape)\n",
    "print(y_train.shape)\n",
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tuned_params = [{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid=[{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = GridSearchCV(LogisticRegression(), tuned_params, scoring = 'accuracy')\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression(C=0.001, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False)\n",
      "0.649387164578\n"
     ]
    }
   ],
   "source": [
    "print(model.best_estimator_)\n",
    "print(model.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.best_estimator_.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64.938716457838566"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_pred, normalize=True) * float(100)\n",
    "acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 9809,  5503],\n",
       "       [ 7341, 13980]], dtype=int64)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L1 regularization and sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_    \n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This means 100 out of 100 are non zero.That means for regularization takes place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see clearly as c decreases sparsity increases.This time 99 features out of 100  are non zero,means 1 features is removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.001, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.00025, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At c=0.00025 we are getting top 2 features and rest becomes zero."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}