Text clasification - one hot encoding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>2000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>4879.877000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>2852.468514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>12.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>2399.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>4841.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>7370.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>9913.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Unnamed: 0\n",
       "count  2000.000000\n",
       "mean   4879.877000\n",
       "std    2852.468514\n",
       "min      12.000000\n",
       "25%    2399.000000\n",
       "50%    4841.500000\n",
       "75%    7370.250000\n",
       "max    9913.000000"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df = pd.read_csv('final.csv')\n",
    "from sklearn.utils import shuffle\n",
    "df = shuffle(df)\n",
    "df = df.head(2000)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400.0\n"
     ]
    }
   ],
   "source": [
    "text_data = df['title']\n",
    "label = df['label']\n",
    "\n",
    "temp = df['label'].value_counts()\n",
    "import numpy as np\n",
    "np.mean(temp)\n",
    "temp\n",
    "print(np.mean(temp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['adult_videos' 'animation_videos' 'education_videos' 'kid_videos'\n",
      " 'sports_videos']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2000, 1592)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "import numpy as np\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# do the lebel part\n",
    "encoder = LabelEncoder()\n",
    "y = encoder.fit_transform(label.astype(str))\n",
    "print(encoder.classes_)\n",
    "\n",
    "\n",
    "# Handle Input text data\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "cv = OneHotEncoder(handle_unknown='ignore')\n",
    "#cv = CountVectorizer()\n",
    "pipeline = Pipeline([\n",
    "\n",
    "    ('tfidf', cv)\n",
    "\n",
    " ])\n",
    "\n",
    "X = pipeline.fit_transform(np.array(text_data).reshape(-1, 1)).toarray()\n",
    "np.shape(X)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([341.,   0., 433.,   0.,   0., 441.,   0., 372.,   0., 413.]),\n",
       " array([0. , 0.4, 0.8, 1.2, 1.6, 2. , 2.4, 2.8, 3.2, 3.6, 4. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.hist(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"vocab.pkl\", 'wb') as handle:\n",
    "        pickle.dump(cv, handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2000, 1492)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca = PCA(n_components=0.95)\n",
    "X_pca = pca.fit_transform(X)\n",
    "np.shape(X_pca)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index 2000 is out of bounds for axis 0 with size 2000",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-f394968cc5c7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mX_mini\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m2000\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mX_mini_PCA\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2000\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[0my_mini\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m2000\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: index 2000 is out of bounds for axis 0 with size 2000"
     ]
    }
   ],
   "source": [
    "#Sample\n",
    "import matplotlib.pyplot as plt\n",
    "X_mini = X[:2000]\n",
    "X_mini_PCA=X[2000]\n",
    "y_mini = y[:2000]\n",
    "\n",
    "plt.hist(y)\n",
    "plt.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SGDClassifier  ( 88%)\n",
    "\n",
    "full dataset\n",
    "\n",
    "clf = SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
    "              early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
    "              l1_ratio=0.15, learning_rate='optimal', loss='hinge',\n",
    "              max_iter=1000, n_iter_no_change=5, n_jobs=None, penalty='l2',\n",
    "              power_t=0.5, random_state=42, shuffle=True, tol=0.001,\n",
    "              validation_fraction=0.1, verbose=0, warm_start=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "clf = SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
    "              early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
    "              l1_ratio=0.15, learning_rate='optimal', loss='hinge',\n",
    "              max_iter=10000, n_iter_no_change=5, n_jobs=None, penalty='l2',\n",
    "              power_t=0.5, random_state=42, shuffle=True, tol=0.001,\n",
    "              validation_fraction=0.1, verbose=0, warm_start=False)\n",
    "# clf.fit(X, y)\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "print(cross_val_score(clf, X_pca, y, cv=5, scoring=\"accuracy\"))\n",
    "print(np.mean(\n",
    "cross_val_score(clf, X_pca, y, cv=10, scoring=\"accuracy\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.77227723 0.745      0.74747475]\n",
      "0.7549173250658399\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "clf = SGDClassifier(random_state=42)\n",
    "#,learning_rate='adaptive',eta0=2\n",
    "param_grid = [\n",
    "            {\n",
    "                'max_iter':list(range(1000,1000000,1000)),\n",
    "           'alpha':  [0.1,.01,0.001,.0001,.00001]   } \n",
    "             ]\n",
    "\n",
    "grid_search = GridSearchCV(clf, param_grid,scoring='accuracy',return_train_score=True)\n",
    "grid_search.fit(X, y)\n",
    "\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "temp = cross_val_score(grid_search, X, y, cv=3, scoring=\"accuracy\")\n",
    "print(temp)\n",
    "print(np.mean(temp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.01, average=False, class_weight=None,\n",
       "              early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
       "              l1_ratio=0.15, learning_rate='optimal', loss='hinge',\n",
       "              max_iter=1000, n_iter_no_change=5, n_jobs=None, penalty='l2',\n",
       "              power_t=0.5, random_state=42, shuffle=True, tol=0.001,\n",
       "              validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'alpha': 0.01, 'max_iter': 1000}"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## svm  (80%)\n",
    "\n",
    "dataset 600\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.80597015 0.85286783 0.8125     0.80952381 0.84170854]\n",
      "0.8245140663830097\n"
     ]
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "clf = svm.SVC(C=1, cache_size=200, class_weight=None, coef0=0.0,\n",
    "    decision_function_shape='ovr', degree=1, gamma='scale', kernel='poly',\n",
    "    max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
    "    tol=0.001, verbose=False)\n",
    "\n",
    "temp = cross_val_score(clf, X_pca, y, cv=5, scoring=\"accuracy\")\n",
    "print(temp)\n",
    "print(np.mean(temp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.78688525 0.71666667 0.75833333 0.78151261 0.78151261]\n",
      "0.7649820911971346\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "param_grid = [\n",
    "                {'C':list(range(1,100)),\n",
    "                 'kernel' : ['linear','poly', 'rbf', 'sigmoid'] ,\n",
    "                'degree':list(range(1,10))\n",
    "                }\n",
    "             \n",
    "             ]\n",
    "\n",
    "grid_search = GridSearchCV(clf, param_grid,scoring='accuracy',return_train_score=True)\n",
    "grid_search.fit(X_mini, y_mini)\n",
    "from sklearn.model_selection import cross_val_score\n",
    "temp = cross_val_score(grid_search, X_mini, y_mini, cv=5, scoring=\"accuracy\")\n",
    "print(temp)\n",
    "print(np.mean(temp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1, cache_size=200, class_weight=None, coef0=0.0,\n",
       "    decision_function_shape='ovr', degree=1, gamma='scale', kernel='poly',\n",
       "    max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dicision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6110002813698865"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import tree\n",
    "clf = tree.DecisionTreeClassifier(random_state=42 , min_samples_leaf=10)\n",
    "from sklearn.model_selection import cross_val_score\n",
    "np.mean(cross_val_score(clf, X_pca, y, cv=5, scoring=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn import tree\n",
    "clf = tree.DecisionTreeClassifier(random_state=42 )\n",
    "\n",
    "param_grid = [\n",
    "             {'min_samples_leaf': list(range(1,100)),\n",
    "              'max_depth':list(range(1,100))}\n",
    "            ]\n",
    "\n",
    "grid_search = GridSearchCV(clf, param_grid,scoring='accuracy',return_train_score=True)\n",
    "grid_search.fit(  X_mini_PCA, y_mini)\n",
    "np.mean(cross_val_score(grid_search,  X_mini_PCA, y_mini, cv=3, scoring=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 86, 'min_samples_leaf': 1}"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8500672818313106"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf = MultinomialNB()\n",
    "clf.fit(X, y)\n",
    "np.mean(cross_val_score(clf,  X, y, cv=10, scoring=\"accuracy\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'GridSearchCV' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-dff3ba7c7400>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      8\u001b[0m             ]\n\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m \u001b[0mgrid_search\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mGridSearchCV\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparam_grid\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mscoring\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'accuracy'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mreturn_train_score\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m \u001b[0mgrid_search\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m  \u001b[0mX_mini_PCA\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_mini\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcross_val_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgrid_search\u001b[0m\u001b[1;33m,\u001b[0m  \u001b[0mX_mini_PCA\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_mini\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"accuracy\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'GridSearchCV' is not defined"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "clf = LogisticRegression(random_state=0)\n",
    "param_grid = [\n",
    "             {\n",
    "                    'max_iter': list(range(1000,100000,10000)),\n",
    "                  'solver': ['lbfgs', 'liblinear' ,'sag','saga']\n",
    "             }\n",
    "            ]\n",
    "\n",
    "grid_search = GridSearchCV(clf, param_grid,scoring='accuracy',return_train_score=True)\n",
    "grid_search.fit(  X_mini_PCA, y_mini)\n",
    "np.mean(cross_val_score(grid_search,  X_mini_PCA, y_mini, cv=3, scoring=\"accuracy\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_iter': 1000, 'solver': 'lbfgs'}"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "C:\\Users\\BC\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8395620559430877"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "clf = LogisticRegression(random_state=0,max_iter= 1000, solver= 'lbfgs')\n",
    "np.mean(cross_val_score(clf,  X_pca, y, cv=10, scoring=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}