models_testing-pretrained_GloVe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model testing using embeddings computed using a pretrained GloVe model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import RidgeClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "from gensim.models import Word2Vec\n",
    "import gensim.downloader as api\n",
    "from helpers import *\n",
    "from preprocessing import *\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Loading model that was pretrained on 2 billion+ tweets._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = api.load('glove-twitter-200')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Let's see how many distinct words we have in the model._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mjouve/opt/anaconda3/envs/ML_Project2/lib/python3.7/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1193514"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model.wv.vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_The vectors are of size 200._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dim = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def toVector(tokens, model, dim):\n",
    "    \"\"\"\n",
    "    This method convert an array of token into a vector of size dim given that word vector in model are of size dim.\n",
    "    \n",
    "    Attributes:\n",
    "    \n",
    "    tokens: arr\n",
    "            tokens to consider for the outputed embedding\n",
    "    \n",
    "    model: gensin model\n",
    "            used to convert a token to a vector of size din\n",
    "            \n",
    "    dim: int \n",
    "            size of the vector    \n",
    "    \"\"\"\n",
    "    \n",
    "    vector = np.zeros(dim)\n",
    "    word_count = 0\n",
    "    \n",
    "    for token in tokens:\n",
    "        if token in model.wv:\n",
    "            vector += model.wv[token]\n",
    "            word_count += 1\n",
    "            \n",
    "    if word_count == 0:\n",
    "        return vector\n",
    "    else:\n",
    "        return vector / float(word_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_For testing purpose let's build model on the subset of tweets provided._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = load_tweets(full = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. tweets with just the tags <user\\> and <url\\> removed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Let's first remove tags and tokenize the tweet._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not . only j...</td>\n",
       "      <td>1</td>\n",
       "      <td>[i, dunno, justin, read, my, mention, or, not,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>because your logic is so dumb , i won't even c...</td>\n",
       "      <td>1</td>\n",
       "      <td>[because, your, logic, is, so, dumb, i, wo, nt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box !  looved the battle ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[just, put, casper, in, a, box, looved, the, b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir &gt; &gt; don't trip lil mama ... just ke...</td>\n",
       "      <td>1</td>\n",
       "      <td>[thanks, sir, do, nt, trip, lil, mama, just, k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting my brother tmr is the bestest birthda...</td>\n",
       "      <td>1</td>\n",
       "      <td>[visiting, my, brother, tmr, is, the, bestest,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity  \\\n",
       "0  i dunno justin read my mention or not . only j...         1   \n",
       "1  because your logic is so dumb , i won't even c...         1   \n",
       "2  just put casper in a box !  looved the battle ...         1   \n",
       "3  thanks sir > > don't trip lil mama ... just ke...         1   \n",
       "4  visiting my brother tmr is the bestest birthda...         1   \n",
       "\n",
       "                                              tokens  \n",
       "0  [i, dunno, justin, read, my, mention, or, not,...  \n",
       "1  [because, your, logic, is, so, dumb, i, wo, nt...  \n",
       "2  [just, put, casper, in, a, box, looved, the, b...  \n",
       "3  [thanks, sir, do, nt, trip, lil, mama, just, k...  \n",
       "4  [visiting, my, brother, tmr, is, the, bestest,...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_1 = remove_tags(tweets)\n",
    "tweets_1 = tokenize_tweets(tweets_1)\n",
    "tweets_1.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Using the tokens we can now compute vector embedding._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mjouve/opt/anaconda3/envs/ML_Project2/lib/python3.7/site-packages/ipykernel_launcher.py:21: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
      "/Users/mjouve/opt/anaconda3/envs/ML_Project2/lib/python3.7/site-packages/ipykernel_launcher.py:22: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n"
     ]
    }
   ],
   "source": [
    "tweets_1['vector'] = tweets_1['tokens'].apply(lambda tokens: toVector(tokens, model, dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vector</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0.08971212630200645, 0.4280543470674235, -0.0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.1027381899766624, 0.19680555940916142, -0.0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[-0.0016766687234242756, 0.33439555598629844, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[-0.09868999943137169, 0.20102208495760956, -0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.2870795548790031, 0.046254554556475744, 0.1...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              vector  polarity\n",
       "0  [0.08971212630200645, 0.4280543470674235, -0.0...         1\n",
       "1  [0.1027381899766624, 0.19680555940916142, -0.0...         1\n",
       "2  [-0.0016766687234242756, 0.33439555598629844, ...         1\n",
       "3  [-0.09868999943137169, 0.20102208495760956, -0...         1\n",
       "4  [0.2870795548790031, 0.046254554556475744, 0.1...         1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_1[['vector', 'polarity']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Let's obtain training and testing data._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_1, test_1 = train_test_split(tweets_1[['vector', 'polarity']], test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157576\n"
     ]
    }
   ],
   "source": [
    "print(len(train_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_1 = np.concatenate(train_1['vector'].values).reshape((len(train_1), dim))\n",
    "y_train_1 = train_1['polarity'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_1 = np.concatenate(test_1['vector'].values).reshape((len(test_1), dim))\n",
    "y_test_1 = test_1['polarity'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will now consider several models for classification."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=500,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_reg_1 = LogisticRegression(solver='lbfgs', max_iter = 500)\n",
    "logistic_reg_1.fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7734934253947302\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = logistic_reg_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C is the inverse of the regularization strenght\n",
    "C = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(C=C)\n",
    "\n",
    "# We first define a new logistic regression model\n",
    "logistic_reg_1 = LogisticRegression(solver='lbfgs', max_iter = 500)\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(logistic_reg_1, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_logistic_1 = clf.fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best C: 21.544346900318832\n"
     ]
    }
   ],
   "source": [
    "print('Best C:', best_logistic_1.best_estimator_.get_params()['C'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7750418845509468\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_logistic_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ridge regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7707772757272681\n"
     ]
    }
   ],
   "source": [
    "ridge_clf_1 = RidgeClassifier().fit(x_train_1, y_train_1)\n",
    "print('Accuracy = {a}'.format(a = ridge_clf_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "ridge_clf_1 = RidgeClassifier()\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(ridge_clf_1, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_ridge_clf_1 = clf.fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 59.94842503189409\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_ridge_clf_1.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7709803523379195\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_ridge_clf_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine\n",
    "\n",
    "_Without hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd_clf_1 = SGDClassifier(loss = 'hinge').fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7706757374219424\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = sgd_clf_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_With hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "svm_clf_1 = SGDClassifier(loss = 'hinge')\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "svm_clf_1 = GridSearchCV(svm_clf_1, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_svm_clf_1 = svm_clf_1.fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 1.0\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_svm_clf_1.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.6060313753363457\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_svm_clf_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. tweets with the tags <user\\> and <url\\> as well as english stop words removed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Let's first remove tags and tokenize the tweets with stop words removal._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not . only j...</td>\n",
       "      <td>1</td>\n",
       "      <td>[dunno, justin, read, mention, justin, god, kn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>because your logic is so dumb , i won't even c...</td>\n",
       "      <td>1</td>\n",
       "      <td>[logic, dumb, wo, nt, even, crop, name, photo,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box !  looved the battle ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[put, casper, box, looved, battle, crakkbitch]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir &gt; &gt; don't trip lil mama ... just ke...</td>\n",
       "      <td>1</td>\n",
       "      <td>[thanks, sir, nt, trip, lil, mama, keep, doin,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting my brother tmr is the bestest birthda...</td>\n",
       "      <td>1</td>\n",
       "      <td>[visiting, brother, tmr, bestest, birthday, gi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity  \\\n",
       "0  i dunno justin read my mention or not . only j...         1   \n",
       "1  because your logic is so dumb , i won't even c...         1   \n",
       "2  just put casper in a box !  looved the battle ...         1   \n",
       "3  thanks sir > > don't trip lil mama ... just ke...         1   \n",
       "4  visiting my brother tmr is the bestest birthda...         1   \n",
       "\n",
       "                                              tokens  \n",
       "0  [dunno, justin, read, mention, justin, god, kn...  \n",
       "1  [logic, dumb, wo, nt, even, crop, name, photo,...  \n",
       "2     [put, casper, box, looved, battle, crakkbitch]  \n",
       "3  [thanks, sir, nt, trip, lil, mama, keep, doin,...  \n",
       "4  [visiting, brother, tmr, bestest, birthday, gi...  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_2 = remove_tags(tweets)\n",
    "tweets_2 = tokenize_tweets(tweets_2, stop_words = True)\n",
    "tweets_2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_We then obtain tweets embeddings._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mjouve/opt/anaconda3/envs/ML_Project2/lib/python3.7/site-packages/ipykernel_launcher.py:21: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
      "/Users/mjouve/opt/anaconda3/envs/ML_Project2/lib/python3.7/site-packages/ipykernel_launcher.py:22: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n"
     ]
    }
   ],
   "source": [
    "tweets_2['vector'] = tweets_2['tokens'].apply(lambda tokens: toVector(tokens, model, dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vector</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0.19875700175762176, 0.6318346992135048, 0.03...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.04200667950014273, 0.19379967575271925, -0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[-0.10455999970436096, 0.3976099997758865, -0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[-0.06771099865436554, 0.172387501783669, -0.2...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.2506899982690811, -0.03688483188549677, 0.0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              vector  polarity\n",
       "0  [0.19875700175762176, 0.6318346992135048, 0.03...         1\n",
       "1  [0.04200667950014273, 0.19379967575271925, -0....         1\n",
       "2  [-0.10455999970436096, 0.3976099997758865, -0....         1\n",
       "3  [-0.06771099865436554, 0.172387501783669, -0.2...         1\n",
       "4  [0.2506899982690811, -0.03688483188549677, 0.0...         1"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_2[['vector', 'polarity']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_2, test_2 = train_test_split(tweets_2[['vector', 'polarity']], test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157576\n"
     ]
    }
   ],
   "source": [
    "print(len(train_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_2 = np.concatenate(train_2['vector'].values).reshape((len(train_2), dim))\n",
    "y_train_2 = train_2['polarity'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_2 = np.concatenate(test_2['vector'].values).reshape((len(test_2), dim))\n",
    "y_test_2 = test_2['polarity'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=1000,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_reg_2 = LogisticRegression(solver='lbfgs', max_iter = 1000)\n",
    "logistic_reg_2.fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7628319033355333\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = logistic_reg_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C is the inverse of the regularization strenght\n",
    "C = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(C=C)\n",
    "\n",
    "logistic_reg_2 = LogisticRegression(solver='lbfgs', max_iter = 500)\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(logistic_reg_2, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_logistic_2 = clf.fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best C: 1291.5496650148827\n"
     ]
    }
   ],
   "source": [
    "print('Best C:', best_logistic_2.best_estimator_.get_params()['C'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7629080570645276\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_logistic_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ridge regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7601919073970655\n"
     ]
    }
   ],
   "source": [
    "ridge_clf_2 = RidgeClassifier().fit(x_train_2, y_train_2)\n",
    "print('Accuracy = {a}'.format(a = ridge_clf_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "ridge_clf_2 = RidgeClassifier()\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(ridge_clf_2, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_ridge_clf_2 = clf.fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 7.742636826811269\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_ridge_clf_2.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7602680611260598\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_ridge_clf_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine\n",
    "\n",
    "_Without hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd_clf_2 = SGDClassifier(loss = 'hinge').fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.761968827740265\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = sgd_clf_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_With hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "svm_clf_2 = SGDClassifier(loss = 'hinge')\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "svm_clf = GridSearchCV(svm_clf_2, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_svm_clf_2 = clf.fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 7.742636826811269\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_svm_clf_2.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7602680611260598\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_svm_clf_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. tweets with tags <user\\> and <url\\> removed and also with word stemming"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_We remove tags, stop words and perform stemming._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not . only j...</td>\n",
       "      <td>1</td>\n",
       "      <td>[i, dunno, justin, read, my, mention, or, not,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>because your logic is so dumb , i won't even c...</td>\n",
       "      <td>1</td>\n",
       "      <td>[becaus, your, logic, is, so, dumb, i, wo, nt,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box !  looved the battle ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[just, put, casper, in, a, box, loov, the, bat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir &gt; &gt; don't trip lil mama ... just ke...</td>\n",
       "      <td>1</td>\n",
       "      <td>[thank, sir, do, nt, trip, lil, mama, just, ke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting my brother tmr is the bestest birthda...</td>\n",
       "      <td>1</td>\n",
       "      <td>[visit, my, brother, tmr, is, the, bestest, bi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity  \\\n",
       "0  i dunno justin read my mention or not . only j...         1   \n",
       "1  because your logic is so dumb , i won't even c...         1   \n",
       "2  just put casper in a box !  looved the battle ...         1   \n",
       "3  thanks sir > > don't trip lil mama ... just ke...         1   \n",
       "4  visiting my brother tmr is the bestest birthda...         1   \n",
       "\n",
       "                                              tokens  \n",
       "0  [i, dunno, justin, read, my, mention, or, not,...  \n",
       "1  [becaus, your, logic, is, so, dumb, i, wo, nt,...  \n",
       "2  [just, put, casper, in, a, box, loov, the, bat...  \n",
       "3  [thank, sir, do, nt, trip, lil, mama, just, ke...  \n",
       "4  [visit, my, brother, tmr, is, the, bestest, bi...  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_3 = remove_tags(tweets)\n",
    "tweets_3 = tokenize_tweets(tweets_3, stop_words = False, stemming = True)\n",
    "tweets_3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mjouve/opt/anaconda3/envs/ML_Project2/lib/python3.7/site-packages/ipykernel_launcher.py:21: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
      "/Users/mjouve/opt/anaconda3/envs/ML_Project2/lib/python3.7/site-packages/ipykernel_launcher.py:22: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n"
     ]
    }
   ],
   "source": [
    "tweets_3['vector'] = tweets_3['tokens'].apply(lambda tokens: toVector(tokens, model, dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vector</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0.09940403963074736, 0.4317200871427422, -0.0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.10893402367623316, 0.17930333719899258, -0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0.014401104715135362, 0.23129667176140678, 0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[-0.09823416670163472, 0.22043208482985696, -0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.2913651106258233, 0.09299010949002372, 0.13...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              vector  polarity\n",
       "0  [0.09940403963074736, 0.4317200871427422, -0.0...         1\n",
       "1  [0.10893402367623316, 0.17930333719899258, -0....         1\n",
       "2  [0.014401104715135362, 0.23129667176140678, 0....         1\n",
       "3  [-0.09823416670163472, 0.22043208482985696, -0...         1\n",
       "4  [0.2913651106258233, 0.09299010949002372, 0.13...         1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_3[['vector', 'polarity']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_3, test_3 = train_test_split(tweets_3[['vector', 'polarity']], test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157576\n"
     ]
    }
   ],
   "source": [
    "print(len(train_3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_3 = np.concatenate(train_3['vector'].values).reshape((len(train_3), dim))\n",
    "y_train_3 = train_3['polarity'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_3 = np.concatenate(test_3['vector'].values).reshape((len(test_3), dim))\n",
    "y_test_3 = test_3['polarity'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=1000,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_reg_3 = LogisticRegression(solver='lbfgs', max_iter = 1000)\n",
    "logistic_reg_3.fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7551657612834441\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = logistic_reg_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C is the inverse of the regularization strenght\n",
    "C = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(C=C)\n",
    "\n",
    "logistic_reg_3 = LogisticRegression(solver='lbfgs', max_iter = 500)\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(logistic_reg_3, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_logistic_3 = clf.fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best C: 2.7825594022071245\n"
     ]
    }
   ],
   "source": [
    "print('Best C:', best_logistic_3.best_estimator_.get_params()['C'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7549880692491242\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_logistic_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ridge regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7521957658526679\n"
     ]
    }
   ],
   "source": [
    "ridge_clf_3 = RidgeClassifier().fit(x_train_3, y_train_3)\n",
    "print('Accuracy = {a}'.format(a = ridge_clf_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "ridge_clf_3 = RidgeClassifier()\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(ridge_clf_3, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_ridge_clf_3 = clf.fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 21.544346900318832\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_ridge_clf_3.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7519419200893537\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_ridge_clf_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine\n",
    "\n",
    "_Without hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd_clf_3 = SGDClassifier(loss = 'hinge').fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7525765344976393\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = sgd_clf_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_With hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "svm_clf_3 = SGDClassifier(loss = 'hinge')\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "svm_clf_3 = GridSearchCV(svm_clf_3, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_svm_clf_3 = clf.fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 21.544346900318832\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_svm_clf_3.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7519419200893537\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_svm_clf_3.score(x_test_3, y_test_3)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}