A detailed notebook (implementation of various algorithms for news classification).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Content\n",
    "* Importing Libraries\n",
    "* Constants\n",
    "* Preprocessing\n",
    "    * Normalizing\n",
    "    * Tokenizing\n",
    "    * Stemming\n",
    "    * Lemmatizing\n",
    "* Feature Engineering\n",
    "    * Bag of Words\n",
    "    * FastText Word2Vec\n",
    "* Model Selection\n",
    "\n",
    "# Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals\n",
    "\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.optimize as opt\n",
    "from functools import reduce\n",
    "from hazm import *\n",
    "from pprint import pprint\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "from sklearn import feature_extraction as fe\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import svm\n",
    "from sklearn import linear_model as l\n",
    "from sklearn import naive_bayes as nb\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from copy import deepcopy\n",
    "\n",
    "# Feature Engineering\n",
    "from sklearn import feature_extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 8, 7, 2, 1, 6, 4, 9, 3])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(list(range(10)))[np.random.permutation(10)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data root path\n",
    "data_root = 'data'\n",
    "\n",
    "# Dataset dataframe column names\n",
    "keys = None\n",
    "\n",
    "# News headline tags\n",
    "valid_tags = None\n",
    "\n",
    "# News agencies\n",
    "news_agencies = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Datapoints: 1000\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(data_root, 'out.jsonl'), encoding='utf-8') as json_data:\n",
    "    news = [json.loads(line) for line in json_data]\n",
    "    news = pd.DataFrame(news)\n",
    "print('Number of Datapoints: {}'.format(len(news)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NewsAgency</td>\n",
       "      <td>_id</td>\n",
       "      <td>body</td>\n",
       "      <td>bodyHtml</td>\n",
       "      <td>date</td>\n",
       "      <td>newsCode</td>\n",
       "      <td>newsLink</td>\n",
       "      <td>newsPath</td>\n",
       "      <td>newsPathLinks</td>\n",
       "      <td>rutitr</td>\n",
       "      <td>subtitle</td>\n",
       "      <td>tags</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0    1     2         3     4         5         6         7   \\\n",
       "0  NewsAgency  _id  body  bodyHtml  date  newsCode  newsLink  newsPath   \n",
       "\n",
       "              8       9         10    11     12  \n",
       "0  newsPathLinks  rutitr  subtitle  tags  title  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keys = list(news.columns)\n",
    "pd.DataFrame([keys])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets look at our data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NewsAgency</th>\n",
       "      <th>body</th>\n",
       "      <th>newsPath</th>\n",
       "      <th>rutitr</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>[مدیرعامل, استقلال, عضو, مدیره, باشگاه, کناره,...</td>\n",
       "      <td>ورزشی</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[استقلال, افتخاری]</td>\n",
       "      <td>[افتخار, قید, ماندن, مدیره, استقلال, زد#زن]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  NewsAgency                                               body newsPath  \\\n",
       "0    AsrIran  [مدیرعامل, استقلال, عضو, مدیره, باشگاه, کناره,...    ورزشی   \n",
       "\n",
       "  rutitr subtitle                tags  \\\n",
       "0   None     None  [استقلال, افتخاری]   \n",
       "\n",
       "                                         title  \n",
       "0  [افتخار, قید, ماندن, مدیره, استقلال, زد#زن]  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### subtitle & rutitr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "473                                                     \n",
      "661    عکس زیر دکتر هاشمی قاضی‌زاده وزیر بهداشت را در...\n",
      "926                                                     \n",
      "372    طولانی شدن غیبت عبدالله دوم شاه اردن پس از دید...\n",
      "297    ظاهرا آنها اعلام کرده‌اند ایران روزانه یک میلی...\n",
      "Name: subtitle, dtype: object\n",
      "939    \n",
      "Name: rutitr, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(news.subtitle.sample(5))\n",
    "print(news.rutitr.sample())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see, there might not exist any 'subtitle' or 'rutitr', so we drop them if they do not have valuable features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not null 'subtitle'  569\n",
      "Not null 'rutitr'  50\n"
     ]
    }
   ],
   "source": [
    "print(\"Not null 'subtitle' \",len([i for i in news.subtitle if len(i) != 0]))\n",
    "print(\"Not null 'rutitr' \",len([i for i in news.rutitr if len(i) != 0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So based on information we got here, we know that these columns can help us, so we consider them."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drop Useless Columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But it is clear for us, `date`,`newsCode`, `newsLink`,`bodyHtml` and `_id` are useless features. So we remove them from our dataset.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "news = news.drop(['_id','date','newsCode','newsLink','bodyHtml'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NewsAgency</th>\n",
       "      <th>body</th>\n",
       "      <th>newsPath</th>\n",
       "      <th>newsPathLinks</th>\n",
       "      <th>rutitr</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>مدیرعامل سابق استقلال از عضویت در هیات مدیره ا...</td>\n",
       "      <td>صفحه نخست » ورزشی</td>\n",
       "      <td>{'صفحه نخست': '/fa/archive?service_id=1', 'ورز...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'استقلال': '/fa/tag/1/استقلال', 'افتخاری': '/...</td>\n",
       "      <td>افتخاری قید ماندن در هیات مدیره استقلال را هم زد</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>دادستان انتظامی مالیاتی سازمان امور مالیاتی گف...</td>\n",
       "      <td>صفحه نخست » اجتماعی</td>\n",
       "      <td>{'صفحه نخست': '/fa/archive?service_id=1', 'اجت...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'مالیات': '/fa/tag/1/مالیات', 'دادستان': '/fa...</td>\n",
       "      <td>دادستان انتظامی مالیاتی سازمان مالیات: آخرین ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>قیمت سبد نفتی اوپک دیروز به روند کاهشی خود ادا...</td>\n",
       "      <td>صفحه نخست » اقتصادی</td>\n",
       "      <td>{'صفحه نخست': '/fa/archive?service_id=1', 'اقت...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'اوپک': '/fa/tag/1/اوپک', 'نفت': '/fa/tag/1/ن...</td>\n",
       "      <td>قیمت سبد نفتی اوپک یک گام دیگر عقب نشست/ 70 دل...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>رئیس فراکسیون فرهنگیان مجلس خطاب به وزیر آموزش...</td>\n",
       "      <td>صفحه نخست » اجتماعی</td>\n",
       "      <td>{'صفحه نخست': '/fa/archive?service_id=1', 'اجت...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'آموزش و پرورش': '/fa/tag/1/آموزش و پرورش', '...</td>\n",
       "      <td>حاجی‌بابایی به بطحایی:آقای وزیر در اطلاع‌رسانی...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>رئیس صندوق بین&amp;zwnj;المللی پول در آستانه نشست ...</td>\n",
       "      <td>صفحه نخست » بین الملل</td>\n",
       "      <td>{'صفحه نخست': '/fa/archive?service_id=1', 'بین...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'صندوق بین المللی پول': '/fa/tag/1/صندوق بین ...</td>\n",
       "      <td>رئیس صندوق بین‌المللی پول: آمریکا از جنگ تعرفه...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  NewsAgency                                               body  \\\n",
       "0    AsrIran  مدیرعامل سابق استقلال از عضویت در هیات مدیره ا...   \n",
       "1    AsrIran  دادستان انتظامی مالیاتی سازمان امور مالیاتی گف...   \n",
       "2    AsrIran  قیمت سبد نفتی اوپک دیروز به روند کاهشی خود ادا...   \n",
       "3    AsrIran  رئیس فراکسیون فرهنگیان مجلس خطاب به وزیر آموزش...   \n",
       "4    AsrIran  رئیس صندوق بین&zwnj;المللی پول در آستانه نشست ...   \n",
       "\n",
       "                newsPath                                      newsPathLinks  \\\n",
       "0      صفحه نخست » ورزشی  {'صفحه نخست': '/fa/archive?service_id=1', 'ورز...   \n",
       "1    صفحه نخست » اجتماعی  {'صفحه نخست': '/fa/archive?service_id=1', 'اجت...   \n",
       "2    صفحه نخست » اقتصادی  {'صفحه نخست': '/fa/archive?service_id=1', 'اقت...   \n",
       "3    صفحه نخست » اجتماعی  {'صفحه نخست': '/fa/archive?service_id=1', 'اجت...   \n",
       "4  صفحه نخست » بین الملل  {'صفحه نخست': '/fa/archive?service_id=1', 'بین...   \n",
       "\n",
       "  rutitr subtitle                                               tags  \\\n",
       "0                  {'استقلال': '/fa/tag/1/استقلال', 'افتخاری': '/...   \n",
       "1                  {'مالیات': '/fa/tag/1/مالیات', 'دادستان': '/fa...   \n",
       "2                  {'اوپک': '/fa/tag/1/اوپک', 'نفت': '/fa/tag/1/ن...   \n",
       "3                  {'آموزش و پرورش': '/fa/tag/1/آموزش و پرورش', '...   \n",
       "4                  {'صندوق بین المللی پول': '/fa/tag/1/صندوق بین ...   \n",
       "\n",
       "                                               title  \n",
       "0   افتخاری قید ماندن در هیات مدیره استقلال را هم زد  \n",
       "1  دادستان انتظامی مالیاتی سازمان مالیات: آخرین ا...  \n",
       "2  قیمت سبد نفتی اوپک یک گام دیگر عقب نشست/ 70 دل...  \n",
       "3  حاجی‌بابایی به بطحایی:آقای وزیر در اطلاع‌رسانی...  \n",
       "4  رئیس صندوق بین‌المللی پول: آمریکا از جنگ تعرفه...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### newsPath & newsPathLinks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "news path links and news path show the same thing? =>  True\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>سلامت</td>\n",
       "      <td>بین الملل</td>\n",
       "      <td>اجتماعی</td>\n",
       "      <td>ورزشی</td>\n",
       "      <td>فناوری و IT</td>\n",
       "      <td>سیاست خارجی</td>\n",
       "      <td>عمومی</td>\n",
       "      <td>علمی</td>\n",
       "      <td>روانشناسی</td>\n",
       "      <td>اقتصادی</td>\n",
       "      <td>فرهنگی/هنری</td>\n",
       "      <td>خواندنی ها و دیدنی ها</td>\n",
       "      <td>سیاسی</td>\n",
       "      <td>حوادث</td>\n",
       "      <td>سرگرمی</td>\n",
       "      <td>دانلود</td>\n",
       "      <td>داستان کوتاه</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0          1        2      3            4            5      6     7   \\\n",
       "0  سلامت  بین الملل  اجتماعی  ورزشی  فناوری و IT  سیاست خارجی  عمومی  علمی   \n",
       "\n",
       "          8        9            10                     11     12     13  \\\n",
       "0  روانشناسی  اقتصادی  فرهنگی/هنری  خواندنی ها و دیدنی ها  سیاسی  حوادث   \n",
       "\n",
       "       14      15            16  \n",
       "0  سرگرمی  دانلود  داستان کوتاه  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newspathlinks_tags = list(set([list(x.keys())[1] for x in news.newsPathLinks]))\n",
    "newspath_tags = list(set(x.split(' » ')[1] for x in news.newsPath))\n",
    "print(\"news path links and news path show the same thing? => \", newspath_tags == newspathlinks_tags)\n",
    "valid_tags = newspath_tags\n",
    "pd.DataFrame([valid_tags])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: we can remove `newsPathLinks` because it is exactly duplicate of `newsPath` and update `newsPath` column in dataframe with just one keyword"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NewsAgency</th>\n",
       "      <th>body</th>\n",
       "      <th>newsPath</th>\n",
       "      <th>rutitr</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>مدیرعامل سابق استقلال از عضویت در هیات مدیره ا...</td>\n",
       "      <td>صفحه نخست » ورزشی</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'استقلال': '/fa/tag/1/استقلال', 'افتخاری': '/...</td>\n",
       "      <td>افتخاری قید ماندن در هیات مدیره استقلال را هم زد</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>دادستان انتظامی مالیاتی سازمان امور مالیاتی گف...</td>\n",
       "      <td>صفحه نخست » اجتماعی</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'مالیات': '/fa/tag/1/مالیات', 'دادستان': '/fa...</td>\n",
       "      <td>دادستان انتظامی مالیاتی سازمان مالیات: آخرین ا...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  NewsAgency                                               body  \\\n",
       "0    AsrIran  مدیرعامل سابق استقلال از عضویت در هیات مدیره ا...   \n",
       "1    AsrIran  دادستان انتظامی مالیاتی سازمان امور مالیاتی گف...   \n",
       "\n",
       "              newsPath rutitr subtitle  \\\n",
       "0    صفحه نخست » ورزشی                   \n",
       "1  صفحه نخست » اجتماعی                   \n",
       "\n",
       "                                                tags  \\\n",
       "0  {'استقلال': '/fa/tag/1/استقلال', 'افتخاری': '/...   \n",
       "1  {'مالیات': '/fa/tag/1/مالیات', 'دادستان': '/fa...   \n",
       "\n",
       "                                               title  \n",
       "0   افتخاری قید ماندن در هیات مدیره استقلال را هم زد  \n",
       "1  دادستان انتظامی مالیاتی سازمان مالیات: آخرین ا...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news = news.drop(['newsPathLinks'], axis=1)\n",
    "news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NewsAgency</th>\n",
       "      <th>body</th>\n",
       "      <th>newsPath</th>\n",
       "      <th>rutitr</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>مدیرعامل سابق استقلال از عضویت در هیات مدیره ا...</td>\n",
       "      <td>ورزشی</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'استقلال': '/fa/tag/1/استقلال', 'افتخاری': '/...</td>\n",
       "      <td>افتخاری قید ماندن در هیات مدیره استقلال را هم زد</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>دادستان انتظامی مالیاتی سازمان امور مالیاتی گف...</td>\n",
       "      <td>اجتماعی</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'مالیات': '/fa/tag/1/مالیات', 'دادستان': '/fa...</td>\n",
       "      <td>دادستان انتظامی مالیاتی سازمان مالیات: آخرین ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>قیمت سبد نفتی اوپک دیروز به روند کاهشی خود ادا...</td>\n",
       "      <td>اقتصادی</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>{'اوپک': '/fa/tag/1/اوپک', 'نفت': '/fa/tag/1/ن...</td>\n",
       "      <td>قیمت سبد نفتی اوپک یک گام دیگر عقب نشست/ 70 دل...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  NewsAgency                                               body newsPath  \\\n",
       "0    AsrIran  مدیرعامل سابق استقلال از عضویت در هیات مدیره ا...    ورزشی   \n",
       "1    AsrIran  دادستان انتظامی مالیاتی سازمان امور مالیاتی گف...  اجتماعی   \n",
       "2    AsrIran  قیمت سبد نفتی اوپک دیروز به روند کاهشی خود ادا...  اقتصادی   \n",
       "\n",
       "  rutitr subtitle                                               tags  \\\n",
       "0                  {'استقلال': '/fa/tag/1/استقلال', 'افتخاری': '/...   \n",
       "1                  {'مالیات': '/fa/tag/1/مالیات', 'دادستان': '/fa...   \n",
       "2                  {'اوپک': '/fa/tag/1/اوپک', 'نفت': '/fa/tag/1/ن...   \n",
       "\n",
       "                                               title  \n",
       "0   افتخاری قید ماندن در هیات مدیره استقلال را هم زد  \n",
       "1  دادستان انتظامی مالیاتی سازمان مالیات: آخرین ا...  \n",
       "2  قیمت سبد نفتی اوپک یک گام دیگر عقب نشست/ 70 دل...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.loc[:,'newsPath'] = list(x.split(' » ')[1] for x in news.newsPath)\n",
    "news.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NewsAgency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AsrIran</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0\n",
       "0  AsrIran"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news_agencies = list(news.NewsAgency.unique())\n",
    "pd.DataFrame([news_agencies])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "not processed tags_dict {'استقلال': '/fa/tag/1/استقلال', 'افتخاری': '/fa/tag/1/افتخاری'}\n",
      "processesd tags_dict ['استقلال', 'افتخاری']\n"
     ]
    }
   ],
   "source": [
    "def tag_extractor(tags_dict):\n",
    "    \"\"\"\n",
    "    gets a tags dictionary and finds unique tags in collection of values and keys\n",
    "    \n",
    "    ::params tags_dict : \n",
    "    \"\"\"\n",
    "    keys = list(set(tags_dict.keys()))\n",
    "    values = { v.split('/')[-1] for v in set(tags_dict.values())}\n",
    "    [values.add(i) for i in keys]\n",
    "    return list(values)\n",
    "\n",
    "print('not processed tags_dict',news.tags[0])\n",
    "print('processesd tags_dict',tag_extractor(news.tags[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we replace `tags` column with extract values from `tag_extractor` function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "news.loc[:, 'tags'] = [tag_extractor(tag) for tag in news.tags]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NewsAgency</th>\n",
       "      <th>body</th>\n",
       "      <th>newsPath</th>\n",
       "      <th>rutitr</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>مدیرعامل سابق استقلال از عضویت در هیات مدیره ا...</td>\n",
       "      <td>ورزشی</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[استقلال, افتخاری]</td>\n",
       "      <td>افتخاری قید ماندن در هیات مدیره استقلال را هم زد</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>دادستان انتظامی مالیاتی سازمان امور مالیاتی گف...</td>\n",
       "      <td>اجتماعی</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[مالیات, دادستان]</td>\n",
       "      <td>دادستان انتظامی مالیاتی سازمان مالیات: آخرین ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>قیمت سبد نفتی اوپک دیروز به روند کاهشی خود ادا...</td>\n",
       "      <td>اقتصادی</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[اوپک, نفت]</td>\n",
       "      <td>قیمت سبد نفتی اوپک یک گام دیگر عقب نشست/ 70 دل...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>رئیس فراکسیون فرهنگیان مجلس خطاب به وزیر آموزش...</td>\n",
       "      <td>اجتماعی</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[آموزش و پرورش, فرهنگیان]</td>\n",
       "      <td>حاجی‌بابایی به بطحایی:آقای وزیر در اطلاع‌رسانی...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>رئیس صندوق بین&amp;zwnj;المللی پول در آستانه نشست ...</td>\n",
       "      <td>بین الملل</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[صندوق بین المللی پول, امریکا]</td>\n",
       "      <td>رئیس صندوق بین‌المللی پول: آمریکا از جنگ تعرفه...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  NewsAgency                                               body   newsPath  \\\n",
       "0    AsrIran  مدیرعامل سابق استقلال از عضویت در هیات مدیره ا...      ورزشی   \n",
       "1    AsrIran  دادستان انتظامی مالیاتی سازمان امور مالیاتی گف...    اجتماعی   \n",
       "2    AsrIran  قیمت سبد نفتی اوپک دیروز به روند کاهشی خود ادا...    اقتصادی   \n",
       "3    AsrIran  رئیس فراکسیون فرهنگیان مجلس خطاب به وزیر آموزش...    اجتماعی   \n",
       "4    AsrIran  رئیس صندوق بین&zwnj;المللی پول در آستانه نشست ...  بین الملل   \n",
       "\n",
       "  rutitr subtitle                            tags  \\\n",
       "0                              [استقلال, افتخاری]   \n",
       "1                               [مالیات, دادستان]   \n",
       "2                                     [اوپک, نفت]   \n",
       "3                       [آموزش و پرورش, فرهنگیان]   \n",
       "4                  [صندوق بین المللی پول, امریکا]   \n",
       "\n",
       "                                               title  \n",
       "0   افتخاری قید ماندن در هیات مدیره استقلال را هم زد  \n",
       "1  دادستان انتظامی مالیاتی سازمان مالیات: آخرین ا...  \n",
       "2  قیمت سبد نفتی اوپک یک گام دیگر عقب نشست/ 70 دل...  \n",
       "3  حاجی‌بابایی به بطحایی:آقای وزیر در اطلاع‌رسانی...  \n",
       "4  رئیس صندوق بین‌المللی پول: آمریکا از جنگ تعرفه...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color='yellow'>Note:</font> If you look at row 0 and row 10, you can see there is many noise in this dataset. We have two different tags for same news."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalizer = Normalizer()\n",
    "news['body'] = news['body'].apply(normalizer.normalize)\n",
    "news['rutitr'] = news['rutitr'].apply(normalizer.normalize)\n",
    "news['subtitle'] = news['subtitle'].apply(normalizer.normalize)\n",
    "news['title'] = news['title'].apply(normalizer.normalize)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(phrase):\n",
    "    sentences = sent_tokenize(phrase)\n",
    "    if len(sentences) > 1:\n",
    "        words = reduce(np.append, [word_tokenize(sentence) for sentence in sentences])\n",
    "    elif len(sentences) == 1:\n",
    "        words = word_tokenize(sentences[0])\n",
    "    else:\n",
    "        words = None\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "news['body'] = news['body'].apply(tokenize)\n",
    "news['rutitr'] = news['rutitr'].apply(tokenize)\n",
    "news['subtitle'] = news['subtitle'].apply(tokenize)\n",
    "news['title'] = news['title'].apply(tokenize)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = Stemmer()\n",
    "stem = lambda s: [stemmer.stem(w) for w in s] if s is not None else None\n",
    "news['body'] = news['body'].apply(stem)\n",
    "news['rutitr'] = news['rutitr'].apply(stem)\n",
    "news['subtitle'] = news['subtitle'].apply(stem)\n",
    "news['title'] = news['title'].apply(stem)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lemmatizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = Lemmatizer()\n",
    "lemmatize = lambda s: [lemmatizer.lemmatize(w) for w in s] if s is not None else None\n",
    "news['body'] = news['body'].apply(lemmatize)\n",
    "news['rutitr'] = news['rutitr'].apply(lemmatize)\n",
    "news['subtitle'] = news['subtitle'].apply(lemmatize)\n",
    "news['title'] = news['title'].apply(lemmatize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NewsAgency</th>\n",
       "      <th>body</th>\n",
       "      <th>newsPath</th>\n",
       "      <th>rutitr</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>[مدیرعامل, سابق, استقلال, از, عضو, در, ه, مدیر...</td>\n",
       "      <td>ورزشی</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[استقلال, افتخاری]</td>\n",
       "      <td>[افتخار, قید, ماندن, در, ه, مدیره, استقلال, را...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>[دادس, انتظام, مالیات, ساز, امور, مالیات, گف, ...</td>\n",
       "      <td>اجتماعی</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[مالیات, دادستان]</td>\n",
       "      <td>[دادس, انتظام, مالیات, ساز, مال, :, آخرین, اخط...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>[قیم, سبد, نفت, اوپک, دیروز, به, روند, کاهش, خ...</td>\n",
       "      <td>اقتصادی</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[اوپک, نفت]</td>\n",
       "      <td>[قیم, سبد, نفت, اوپک, یک, گا, دیگر, عقب, نشست/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>[رئیس, فراکسیون, فرهنگ, مجلس, خطاب, به, وزیر, ...</td>\n",
       "      <td>اجتماعی</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[آموزش و پرورش, فرهنگیان]</td>\n",
       "      <td>[حاجی‌بابا, به, بطحا, :, آقا, وزیر, در, اطلاع‌...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>[رئیس, صندوق, بین&amp;zwnj;الملل, پول, در, آستانه,...</td>\n",
       "      <td>بین الملل</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[صندوق بین المللی پول, امریکا]</td>\n",
       "      <td>[رئیس, صندوق, بین‌الملل, پول, :, آمریکا, از, ج...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  NewsAgency                                               body   newsPath  \\\n",
       "0    AsrIran  [مدیرعامل, سابق, استقلال, از, عضو, در, ه, مدیر...      ورزشی   \n",
       "1    AsrIran  [دادس, انتظام, مالیات, ساز, امور, مالیات, گف, ...    اجتماعی   \n",
       "2    AsrIran  [قیم, سبد, نفت, اوپک, دیروز, به, روند, کاهش, خ...    اقتصادی   \n",
       "3    AsrIran  [رئیس, فراکسیون, فرهنگ, مجلس, خطاب, به, وزیر, ...    اجتماعی   \n",
       "4    AsrIran  [رئیس, صندوق, بین&zwnj;الملل, پول, در, آستانه,...  بین الملل   \n",
       "\n",
       "  rutitr subtitle                            tags  \\\n",
       "0   None     None              [استقلال, افتخاری]   \n",
       "1   None     None               [مالیات, دادستان]   \n",
       "2   None     None                     [اوپک, نفت]   \n",
       "3   None     None       [آموزش و پرورش, فرهنگیان]   \n",
       "4   None     None  [صندوق بین المللی پول, امریکا]   \n",
       "\n",
       "                                               title  \n",
       "0  [افتخار, قید, ماندن, در, ه, مدیره, استقلال, را...  \n",
       "1  [دادس, انتظام, مالیات, ساز, مال, :, آخرین, اخط...  \n",
       "2  [قیم, سبد, نفت, اوپک, یک, گا, دیگر, عقب, نشست/...  \n",
       "3  [حاجی‌بابا, به, بطحا, :, آقا, وزیر, در, اطلاع‌...  \n",
       "4  [رئیس, صندوق, بین‌الملل, پول, :, آمریکا, از, ج...  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter Words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove Stopwords\n",
    "For this step, we use stopwords from <a href='https://github.com/kharazi/persian-stopwords'>this repository</a>.\n",
    "\n",
    "There are some files of stopwords and we are using <a href='https://github.com/kharazi/persian-stopwords/blob/master/persian'>this</a>.\n",
    "\n",
    "```bash\n",
    "git clone https://github.com/kharazi/persian-stopwords.git\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>اید</td>\n",
       "      <td>ایشان</td>\n",
       "      <td>ایم</td>\n",
       "      <td>این</td>\n",
       "      <td>این جوری</td>\n",
       "      <td>این قدر</td>\n",
       "      <td>این گونه</td>\n",
       "      <td>اینان</td>\n",
       "      <td>اینجا</td>\n",
       "      <td>اینجاست</td>\n",
       "      <td>ایند</td>\n",
       "      <td>اینطور</td>\n",
       "      <td>اینقدر</td>\n",
       "      <td>اینها</td>\n",
       "      <td>اینهاست</td>\n",
       "      <td>اینو</td>\n",
       "      <td>اینچنین</td>\n",
       "      <td>اینک</td>\n",
       "      <td>اینکه</td>\n",
       "      <td>اینگونه</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0      1    2    3         4        5         6      7      8        9   \\\n",
       "0  اید  ایشان  ایم  این  این جوری  این قدر  این گونه  اینان  اینجا  اینجاست   \n",
       "\n",
       "     10      11      12     13       14    15       16    17     18       19  \n",
       "0  ایند  اینطور  اینقدر  اینها  اینهاست  اینو  اینچنین  اینک  اینکه  اینگونه  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords_root = 'persian-stopwords'\n",
    "\n",
    "with open(os.path.join(stopwords_root, 'persian'), encoding='utf-8') as stopwords_file:\n",
    "    stopwords = [re.sub(r'\\n','',word) for word in stopwords_file]\n",
    "pd.DataFrame([stopwords[150:170]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we remove all stopwords from our dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['افتخار', 'قید', 'ماندن', 'در', 'ه', 'مدیره', 'استقلال', 'را', 'ه', 'زد#زن']\n",
      "['افتخار', 'قید', 'ماندن', 'مدیره', 'استقلال', 'زد#زن']\n"
     ]
    }
   ],
   "source": [
    "def filter_words(words_list, stopwords=stopwords):\n",
    "    \"\"\"\n",
    "    Gets a list of words and remove stopwords from that list.\n",
    "    \n",
    "    :param words_list: a list of words to apply stopwords\n",
    "    :param stopwords: a list of stopwords to remove from words_list\n",
    "    \"\"\"\n",
    "    if words_list is None:\n",
    "        return None\n",
    "    filtered_words = [word for word in words_list if word not in stopwords]\n",
    "    return filtered_words\n",
    "\n",
    "s = news.title[0]\n",
    "print(s)\n",
    "s_filtered = filter_words(s, stopwords)\n",
    "print(s_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "news['body'] = news['body'].apply(filter_words)\n",
    "news['rutitr'] = news['rutitr'].apply(filter_words)\n",
    "news['subtitle'] = news['subtitle'].apply(filter_words)\n",
    "news['title'] = news['title'].apply(filter_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['افتخار', 'قید', 'ماندن', 'مدیره', 'استقلال', 'زد#زن']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NewsAgency</th>\n",
       "      <th>body</th>\n",
       "      <th>newsPath</th>\n",
       "      <th>rutitr</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>[مدیرعامل, استقلال, عضو, مدیره, باشگاه, کناره,...</td>\n",
       "      <td>ورزشی</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[استقلال, افتخاری]</td>\n",
       "      <td>[افتخار, قید, ماندن, مدیره, استقلال, زد#زن]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>[دادس, انتظام, مالیات, ساز, مالیات, گف, صور, ع...</td>\n",
       "      <td>اجتماعی</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[مالیات, دادستان]</td>\n",
       "      <td>[دادس, انتظام, مالیات, ساز, مال, آخرین, اخطار,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AsrIran</td>\n",
       "      <td>[قیم, سبد, نفت, اوپک, روند, کاهش, ادامه, سه, ش...</td>\n",
       "      <td>اقتصادی</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[اوپک, نفت]</td>\n",
       "      <td>[قیم, سبد, نفت, اوپک, گا, نشست/, ۷۰, دلار, بشکه]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  NewsAgency                                               body newsPath  \\\n",
       "0    AsrIran  [مدیرعامل, استقلال, عضو, مدیره, باشگاه, کناره,...    ورزشی   \n",
       "1    AsrIran  [دادس, انتظام, مالیات, ساز, مالیات, گف, صور, ع...  اجتماعی   \n",
       "2    AsrIran  [قیم, سبد, نفت, اوپک, روند, کاهش, ادامه, سه, ش...  اقتصادی   \n",
       "\n",
       "  rutitr subtitle                tags  \\\n",
       "0   None     None  [استقلال, افتخاری]   \n",
       "1   None     None   [مالیات, دادستان]   \n",
       "2   None     None         [اوپک, نفت]   \n",
       "\n",
       "                                               title  \n",
       "0        [افتخار, قید, ماندن, مدیره, استقلال, زد#زن]  \n",
       "1  [دادس, انتظام, مالیات, ساز, مال, آخرین, اخطار,...  \n",
       "2   [قیم, سبد, نفت, اوپک, گا, نشست/, ۷۰, دلار, بشکه]  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(news.title[0])\n",
    "news.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['IObit', 'Malware', 'Fighter', 'برنامه', 'عال', 'بردن', 'افزار', 'نر', 'افزار', 'جاسوس', 'ابزار', 'تبلیغات', 'مزاح', 'تروجان', 'لاگر', 'ربات', 'کرم', 'بود#باش', 'موتور', 'هسته', 'تعبیه', 'نر', 'افزار', 'IObit', 'Malware', 'Fighter', 'توانست#توان', 'برنامه', 'مخرب', 'اس', 'سیس', 'آسیب', 'رساند#رسان', 'ببرید', 'کوچک', 'مشکل', 'سیس', 'ایجاد', 'سیس', 'شد#شو', 'نر', 'افزار', 'IObit', 'Malware', 'Fighter', 'بردن', 'افزار', 'داشت#دار', 'اسکن', 'هوشمند', 'اسکن', 'اسکن', 'سفارش', '&nbsp;و', 'مطمئن', 'بود#باش', 'سه', 'حرفه', 'عمق', 'بدافزار', 'سیس', 'نابود', 'شد#شو', 'قابلیت', 'نر', 'افزار', 'IObit', 'Malware', 'Fighter', 'امک', 'رس', 'خودکار', 'نر', 'افزار', 'اینترن', 'امک', 'اسکن', 'سیس', 'سه', 'متد', 'اسکن', 'سیس', 'صور', 'مداو', 'اتوماتیک', 'جلوگیر', 'نفوذ', 'برنامه', 'مخرب', 'سیس', 'ایجاد', 'Real-time', 'Protection', 'قو', 'قدرتمند', 'ابزار', 'Startup', 'Guard', 'Browser', 'Guard', 'Network', 'Guard', 'File', 'Guard', 'Cookie', 'Guard', 'Process', 'Guard', 'USD', 'Disk', 'Guard', 'Malicious', 'Action', 'Guard', 'توانا', 'عمیق', 'فایل', 'مخرب', 'فناور', 'DOG', 'تامین', 'مراقبت', 'عل', 'Anti-malware', 'Anti-spyware', 'Anti-adware', 'Anti-trojan', 'Anti-bots', 'سبک', 'نر', 'افزار', 'سبب', 'موتور', 'جس', 'جو', 'بهینه', 'هسته', 'دارا', 'آخرین', 'فن', 'محاسب', 'ابر', 'تجزیه', 'تحلیل', 'رفتار', 'انواع', 'فایل', 'سازگار', 'انواع', 'ویروس', 'بسته', 'امنیت', 'دانلود', 'نر', 'افزار&nbsp;IObit', 'Malware', 'Fighter']\n",
      "['Magic', 'Tiles', '&ndash;', 'کاشی', 'جادو', '۳&nbsp;باز', 'محبوب', 'سرگر', 'سبک', 'بازی', 'موزیکال', 'استودیو', 'ساز', 'Amanotes', 'JSC', 'اندروید', 'اس', 'صور', 'رایگ', 'گوگل', 'پل', 'عرضه', 'لحظه', 'ب', 'کاربر', 'اندروید', 'جه', 'دریاف', 'محبوب', 'بازی', 'دسته', 'بند', 'موزیکال', 'شمار', 'رفت#رو', '!', 'ال', 'محبوب', 'نا', 'آشنا', '&ldquo;Piano', 'Tiles', '۲&nbsp;&rdquo;', 'ساخته_شده_اس', 'امک', 'نواختن', 'موزیک', 'پیانو', 'داد#ده', 'سرع', 'کاشی', 'موجود', 'صفحه', 'نما', 'لمس', 'کرد#کن', 'اقدا', 'نواختن', 'موزیک', 'نمود#نما', 'ساعت', 'مشغول', 'بود#باش', '!', 'کاشی', 'صفحه', 'نما', 'سم', 'پایین', 'حرک', '#هست', 'پایین', 'خارج', 'شد#شو', 'لمس', 'کرد#کن', '!', 'افزا', 'تعداد', 'کاشی', 'خروج', 'سرع', 'خروج', 'کاشی', 'افزا', 'کرد#کن', 'هیج', 'شد#شو', '!', 'کاشی', 'مشابه', 'کاشی', 'موزیک', 'دلربا', 'قرار', 'خواهند_گرف', 'صور', 'طول', 'لمس', 'کرد#کن', '!', 'علاقه', 'مند', 'بازی', 'موزیکال', '#هست', 'شک', 'Magic', 'Tiles', 'طراح', 'ساخ', 'عال', 'نظر', 'جلب', 'نمود#نما', '!', 'دانلود', 'Magic', 'Tiles']\n",
      "['Top', 'Drives', '&ndash;', 'تاپ', 'درایوز&nbsp;باز', 'محبوب', 'سرگر', 'جالب', 'سبک&nbsp;بازی', 'ماشین', 'سواری&nbsp;&ndash;', 'مدیر', 'ماشین', 'کارت', 'استودیو', 'بازیساز', 'Hutch', 'Games', 'اندروید', 'اس', 'Top', 'Drives', 'قادرید', 'کار', '', 'ب', '۷۰۰', 'کار', 'برد#بر', 'شناخته', 'ماشین', 'جمله', 'پورشه', 'ب', 'دبلیو', 'غیره', 'انتخاب', 'کرد#کن', 'مجموعه', 'کارت', 'داشت#دار', '!', 'کارت', 'مسابقه', 'اتومبیل', 'ران', 'دنیا', 'شرک', 'کرد#کن', 'بخ', 'برنده', 'بیازمایید', '!', 'حالت', 'امک', 'سرگر', 'داد#ده', 'عنو', 'مثال', 'نفره', 'آنلاین', 'کاربر', 'جهان', 'رقاب', 'کرد#کن', 'صور', 'پیروز', 'اس', 'کار', 'ماشین', 'خاص', 'دس', 'کرد#کن', '!', 'امک', 'مدیر', 'ماشین', 'حت', 'ارتقا', 'امک', 'شرک', 'مسابق', 'رانندگ', 'مسابق', 'شتاب', 'مسابق', 'سرع', 'پیس', 'مسابق', 'رفتن', 'تپه', 'لحاظ', 'شرایط', 'آب', 'هوا', 'سطوح', 'جاده', 'ساخ', 'جالب', 'توانست#توان', 'نظر', 'دوستدار', 'بازی', 'کارت', 'جلب', '!', 'دنبال', 'سرگر', 'پر', 'اوق', 'فراغ', '', '#هست', 'Top', 'Drives', 'امتح', 'کرد#کن', '!', 'نک', 'تکمیل', '&nbsp;باز', 'آنلاین', 'اینترنت', 'استدو', '&nbsp;باز', 'هک', 'نمیشود', 'نسخه', 'مود', 'نداردسه', '&nbsp;جدید', 'آپد', 'همز', 'انتشار', 'صفحه', 'دریاف', 'کرد#کن', '&nbsp;', 'دستورالعمل', 'نصب', 'اجرا', '&ndash;', 'ابتدا', 'فایل', 'نصب', 'دانلود', 'نصب', 'کرد#کن', '&ndash;', 'فایل', 'دیتا', 'دانلود', '', 'فشرده', 'خارج', 'کرد#کن', 'پوشه', 'com', 'hutchgames', 'cccg', 'Android/Obb', 'حافظه', 'دستگاه', 'کپ', 'کرد#کن', '&ndash;', 'آنلاین', 'اجرا', 'نمود#نما', 'دانلود', 'بازی&nbsp;Top', 'Drives', 'اندروید', 'دانلود', 'دیتا', 'بازی&nbsp;Top', 'Drives', 'منبع', 'فارسروید']\n",
      "['DeskCalc&nbsp;تما', 'ویژگی', 'ماشین', 'حساب', 'معمول', 'علاوه', 'کاربر', 'ماشین', 'حساب', 'دیجیتال', 'تحت&nbsp;ویندوز&nbsp;انتظار', 'داشت#دار', 'گنجانده', 'اس', 'ویندوز', 'ماشین', 'حساب', 'کاره', 'انجا', 'محاسب', 'عدد', 'ساده', 'محاسبه', 'فرمول', 'ریاض', 'دشوار', 'کاربا', 'انواع', 'حسابدار', 'مهندس', '…داشته', 'بود#باش', 'ویژگ', '', 'افزودن', 'ورود', 'منطق', 'ذخیره', 'بازگردان', 'محاسب', 'انجا', 'چاپ', 'محاسب', 'جداساز', 'ارقا', 'کار', 'اعشار', 'کار', 'فرمول', 'مهندس', 'تصحیح', 'رس', 'اضافه', 'حذف', 'ارز', 'تهیه', 'خروج', 'اکسل', 'تن', 'برخ', 'امک', 'ماشین', 'حسابنر', 'افزار', 'بود#باش', 'قابلیت', 'کلیدی&nbsp;نر', 'افزار&nbsp;DeskCalc', 'وارد', 'متون', 'ارگونامیک-', 'ویژگی', 'تصحیح', 'رس', 'اضافه', 'حذف', 'ارز', 'کار', 'رق', 'اعشار', 'شناور-', 'مفسر', 'فرمول-', 'توابع', 'محاسبه', 'مال', 'فروش-', 'محاسبه', 'درصد-', 'مبدل', 'ارز-', 'توابع', 'مثلثات', 'سینوس', 'کسینوس', 'تانژان', 'رابط', 'کاربر', 'مشابه', 'نر', 'افزار', 'آفیس-', 'خروج', 'Excel-', 'نما', 'پاسخ', 'خط', 'عنو', 'پنجره-', 'دانلود', 'نر', 'افزار&nbsp;DeskCalc']\n",
      "['Elite', 'Trials&nbsp;نام&nbsp;بازی&nbsp;ا', 'زیبا', 'سرگر', 'سبک', 'بازی', 'موتور', 'سوار', 'مسابقه', 'استودیو', 'ساز', 'GX', 'Games', 'بود#باش', 'برای&nbsp;اندروید&nbsp;منتشر', 'شده_اس', 'موتور', 'متنوع', 'انتخاب', 'داشت#دار', 'شامل', 'نفره', 'نفره', 'آنلاین', 'بود#باش', 'مهارت', 'موتورسوار', 'داد#ده', 'موانع', 'خطرناک', 'شیشه', 'خورده', 'موانع', 'نوک', 'تیز', 'بمب', 'جر', 'هوا', 'عبور', 'کرد#کن', 'زیبا', 'مهیج', 'ترین&nbsp;بازی', 'موتورسوار', 'لذ', 'ببرید', 'امتیاز', '۴٫۶', 'کسب', 'دارای&nbsp;گرافیک&nbsp;زیبا', 'جذاب', 'همراه', 'پل', 'اعتیادآور', 'بود#باش', 'زیبا', 'افزوده', 'ساعت', 'سرگر', 'دانلود', 'بازی&nbsp;Elite', 'Trials', 'اندروید']\n"
     ]
    }
   ],
   "source": [
    "for idx,n in enumerate(news.newsPath):\n",
    "    if n=='دانلود':\n",
    "        print(news.body[idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "this shows that our 'دانلود' newsPath tag is valid but there is only a few samples for this tag."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove ASCII\n",
    "There are a lot of ascii characters which we do not need them such as html tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_ascii(word_list):\n",
    "    \"\"\"\n",
    "    Remove ascii characters from a list of words or a string\n",
    "    :params word_list: a list of string or a string\n",
    "    \"\"\"\n",
    "    \n",
    "    if type(word_list) is list:\n",
    "        return [re.sub(r'[\\x00-\\x7F]+',' ',w) for w in word_list]\n",
    "    if type(word_list) is str:\n",
    "        words = word_list.split()\n",
    "        return [re.sub(r'[\\x00-\\x7F]+',' ',w) for w in words]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering\n",
    "The inherent unstructured (no neatly formatted data columns!) and noisy nature of textual data makes it harder for machine learning methods to directly work on raw text data.\n",
    "\n",
    "##### Motivation\n",
    "The importance of feature engineering is even more important for unstructured, textual data because we need to convert free flowing text into some numeric representations which can then be understood by machine learning algorithms."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Engineering Strategies\n",
    "We try different methods and compare result using F1, precision and recall score gathered by result of different machine learning methods."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Libraries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bag of Words Model\n",
    "First of all we should define **vector space** term. When we convert unstructured data to number such each dimension of a vector is one feature of space, we have a vector space.\n",
    "\n",
    "**Bag of words** model is one of simpleset vector space methods.<br>\n",
    "In bag of words model, each vector represent a document in corpus and each dimension is word in document and the value of corresponding dimension is the frequency of given word."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating the trainable Document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_count = 2\n",
    "\n",
    "body_train = []\n",
    "for i, w in news.iterrows():\n",
    "    body_train.append([])\n",
    "    if w.body is not None:\n",
    "        body_train[i] += w.body\n",
    "    if w.title is not None:\n",
    "        body_train[i] += w.title * title_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n"
     ]
    }
   ],
   "source": [
    "print(len([w for w in body_train if (w is not None) and len(w) != 0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding Unknown Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_uknown(train_doc, thresh= 2):\n",
    "    vocab = list(set(reduce(lambda x, y: np.append(x, y), train_doc)))\n",
    "    vocab = dict.fromkeys(vocab, 0)\n",
    "    mega_doc = reduce(np.append, train_doc)\n",
    "    for w in mega_doc:\n",
    "        vocab[w] += 1\n",
    "    unknown_list = []\n",
    "    for k, v in vocab.items():\n",
    "        if v < thresh:\n",
    "            unknown_list.append(k)\n",
    "    unknown_list = set(unknown_list)\n",
    "    for i, doc in enumerate(train_doc):\n",
    "        for j, w in enumerate(doc):\n",
    "            if w in unknown_list:\n",
    "                train_doc[i][j] = 'unk_w'\n",
    "    return train_doc\n",
    "body_train = make_uknown(body_train, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### convert a list of string items to a string item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert a list of string items to a string item (detokenization)\n",
    "\n",
    "        \n",
    "# just check it out! I got a weird result!\n",
    "# body_train = [' '.join([w for w in s]) for s in body_train if s is not None]\n",
    "body_train_ = []\n",
    "for idx,s in enumerate(body_train):\n",
    "    string=''\n",
    "    if s is not None:\n",
    "        for w in s:\n",
    "            string+=' '+w\n",
    "        body_train_.append(string)\n",
    "    else:\n",
    "        body_train_.append(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note: We just apply this method to \"body\", then after builing some models, we try other ones.\n",
    "def bag_of_words(train):\n",
    "    \"\"\"\n",
    "    calculate bag of word vector space of a list of strings\n",
    "    \n",
    "    :params train: train data as a list of strings\n",
    "    \"\"\"\n",
    "    cv = fe.text.CountVectorizer(ngram_range=(1, 4), min_df=0.005, vocabulary=None,\n",
    "                                lowercase=False, analyzer='word') # 4-gram model\n",
    "    bow_train = cv.fit_transform(train)\n",
    "    return bow_train, cv.get_feature_names()\n",
    "bowt,feature_names = bag_of_words(body_train_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FATF</th>\n",
       "      <th>ad_type</th>\n",
       "      <th>ad_type standardvideo</th>\n",
       "      <th>ad_type standardvideo max_ad_duration</th>\n",
       "      <th>ad_type standardvideo max_ad_duration ۶۰۰۰</th>\n",
       "      <th>ads</th>\n",
       "      <th>ads pre</th>\n",
       "      <th>ads pre roll</th>\n",
       "      <th>ads pre roll advert</th>\n",
       "      <th>advert</th>\n",
       "      <th>...</th>\n",
       "      <th>۹۲</th>\n",
       "      <th>۹۳</th>\n",
       "      <th>۹۴</th>\n",
       "      <th>۹۵</th>\n",
       "      <th>۹۵ درصد</th>\n",
       "      <th>۹۶</th>\n",
       "      <th>۹۷</th>\n",
       "      <th>۹۸</th>\n",
       "      <th>۹۹</th>\n",
       "      <th>۹۹ درصد</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 10865 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   FATF  ad_type  ad_type standardvideo  \\\n",
       "0     0        0                      0   \n",
       "1     0        0                      0   \n",
       "2     0        0                      0   \n",
       "3     0        0                      0   \n",
       "4     0        0                      0   \n",
       "\n",
       "   ad_type standardvideo max_ad_duration  \\\n",
       "0                                      0   \n",
       "1                                      0   \n",
       "2                                      0   \n",
       "3                                      0   \n",
       "4                                      0   \n",
       "\n",
       "   ad_type standardvideo max_ad_duration ۶۰۰۰  ads  ads pre  ads pre roll  \\\n",
       "0                                           0    0        0             0   \n",
       "1                                           0    0        0             0   \n",
       "2                                           0    0        0             0   \n",
       "3                                           0    0        0             0   \n",
       "4                                           0    0        0             0   \n",
       "\n",
       "   ads pre roll advert  advert   ...     ۹۲  ۹۳  ۹۴  ۹۵  ۹۵ درصد  ۹۶  ۹۷  ۹۸  \\\n",
       "0                    0       0   ...      0   0   0   0        0   0   0   0   \n",
       "1                    0       0   ...      0   0   0   0        0   0   0   0   \n",
       "2                    0       0   ...      0   0   0   0        0   0   0   0   \n",
       "3                    0       0   ...      0   0   0   0        0   0   2   0   \n",
       "4                    0       0   ...      0   0   0   0        0   0   0   0   \n",
       "\n",
       "   ۹۹  ۹۹ درصد  \n",
       "0   0        0  \n",
       "1   0        0  \n",
       "2   0        0  \n",
       "3   0        0  \n",
       "4   0        0  \n",
       "\n",
       "[5 rows x 10865 columns]"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(bowt.todense(),columns=feature_names).head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Bag of Words Vector Space Dimensions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of examples: 1000\n",
      "number of features: 10865\n"
     ]
    }
   ],
   "source": [
    "print('number of examples: {}'.format(bowt.shape[0]))\n",
    "print('number of features: {}'.format(bowt.shape[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "OK let it go!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Labels\n",
    "Now for training purpose, we should train our model using train data and corresponding labels.\n",
    "\n",
    "In this dataset, labels are `newsPath`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To deal with multiclass labels, we use `onehotencoders` to encode classes into categorical values represted by numbers.\n",
    "Here we go..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>newsPath</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   newsPath\n",
       "0        16\n",
       "1         0\n",
       "2         1\n",
       "3         0\n",
       "4         2"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = news['newsPath'].values\n",
    "\n",
    "def label_encoder(array):\n",
    "    \"\"\"\n",
    "    Return corresponding label encoded array\n",
    "    \"\"\"\n",
    "    array_label_encoder = LabelEncoder()\n",
    "    encoded = array_label_encoder.fit_transform(array)\n",
    "    return array_label_encoder,encoded\n",
    "    \n",
    "def onehot_encoder(label_encoded_array):\n",
    "    \"\"\"\n",
    "    Return onehot encoded version of a label encoded input array\n",
    "    \"\"\"\n",
    "    array_onehot_encoder = OneHotEncoder()\n",
    "    onehot_encoded_array = array_onehot_encoder.fit_transform(label_encoded_array)\n",
    "    return array_onehot_encoder, onehot_encoded_array\n",
    "\n",
    "def one_hot_encoder(labels):\n",
    "    label_set = np.array(list(set(labels)))\n",
    "    en_labels = np.array(list(map(lambda x: x == label_set, labels)))\n",
    "    return label_set, en_labels\n",
    "\n",
    "pd.DataFrame(label_encoder(y)[1], columns=['newsPath']).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Features for Naive Bayes Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Erfan\\Anaconda3\\envs\\tf\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2069: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "y_decoder, y_en = one_hot_encoder(y)\n",
    "x_train_naive,x_test_naive ,y_train_naive ,y_test_naive = train_test_split(body_train , y_en, train_size = 0.8, random_state=85)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Split Dataset Into Trainset and Testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for Naive Bayes we just need to label encode target labels.\n",
    "_, y_label_encoded = label_encoder(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2069: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x #examples: 1000, x #features:10865\n",
      "x_train #examples: 900, x_train #features: 10865\n",
      "x_test #examples: 100, x_test #features: 10865\n",
      "y_train #examples: 900, y_train #features: 1\n",
      "y_test #examples: 100, y_test #features: 1\n"
     ]
    }
   ],
   "source": [
    "x_train,x_test,y_train,y_test = train_test_split(bowt , y_label_encoded.reshape(-1,1), train_size = 0.90, random_state=85)\n",
    "print('x #examples: {}, x #features:{}'.format(bowt.shape[0], bowt.shape[1]))\n",
    "print('x_train #examples: {}, x_train #features: {}'.format(x_train.shape[0], x_train.shape[1]))\n",
    "print('x_test #examples: {}, x_test #features: {}'.format(x_test.shape[0], x_test.shape[1]))\n",
    "print('y_train #examples: {}, y_train #features: {}'.format(y_train.shape[0], y_train.shape[1]))\n",
    "print('y_test #examples: {}, y_test #features: {}'.format(y_test.shape[0], y_test.shape[1]))\n",
    "\n",
    "x_train = x_train.todense()\n",
    "x_test = x_test.todense()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy_on_cm(confusion_matrix):\n",
    "    \"\"\"\n",
    "    Calculate accuracy on given confusion matrix\n",
    "    \"\"\"\n",
    "    t = np.trace(confusion_matrix)\n",
    "    f = np.sum(confusion_matrix) - t\n",
    "    ac = t/(t+f)\n",
    "    return (t,f,ac)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NaiveBayes:\n",
    "    def __init__(self, nb_type='multiclass_nb'):\n",
    "        pass\n",
    "\n",
    "    def fit(self, documents, labels, s_value = 0.1):\n",
    "        \"\"\"[This function will create a naive bayes model and fit it to the documents and labels]\n",
    "        \n",
    "        Arguments:\n",
    "            documents {[unstructured array]} -- [...]\n",
    "            labels {[2_D array]} -- [One hot encoded array]\n",
    "            s_value {[float]} -- [Smoothing Value]\n",
    "        \"\"\"\n",
    "        labels = np.array(labels)\n",
    "        documents = np.array(documents)\n",
    "        self.class_probs = np.zeros(labels.shape[1])\n",
    "\n",
    "        # Calculating P(c)\n",
    "        for label in labels:\n",
    "            self.class_probs[np.argmax(label)] += 1\n",
    "        self.class_probs = np.log(self.class_probs/labels.shape[0])\n",
    "\n",
    "        # Creating Vocabulary\n",
    "        self.vocab = list(set(reduce(lambda x, y: np.append(x, y), documents)))\n",
    "            \n",
    "        # Calculating P(w|c)\n",
    "        self.mega_probs = [None] * labels.shape[1]\n",
    "        for i in range(labels.shape[1]):\n",
    "            mega_doc = reduce(np.append, documents[np.array(list(map(np.argmax, labels))) == i])\n",
    "            self.mega_probs[i] = dict.fromkeys(self.vocab, s_value)\n",
    "            words_count = 0\n",
    "            for w in mega_doc:\n",
    "                words_count += 1\n",
    "                self.mega_probs[i][w] += 1\n",
    "            self.mega_probs[i] = {k:np.log(v/((len(self.vocab)*s_value) + words_count)) for k, v in self.mega_probs[i].items()}\n",
    "            \n",
    "    def predict(self, documents):\n",
    "        pred_labels = []\n",
    "        for doc in documents:\n",
    "            probs = deepcopy(self.class_probs[:])\n",
    "            for i in range(len(probs)):\n",
    "                for w in doc:\n",
    "                    if w in self.vocab:\n",
    "                        probs[i] += self.mega_probs[i][w]\n",
    "                    else:\n",
    "                        probs[i] += self.mega_probs[i]['unk_w']\n",
    "            pred_labels.append(probs)\n",
    "        return pred_labels\n",
    "    \n",
    "    def evaluate(self, documents, labels):\n",
    "        pred_labels = self.predict(documents)\n",
    "        print('Accuracy: {}'.format(np.mean(np.argmax(pred_labels, 1) == np.argmax(labels, 1))))        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [],
   "source": [
    "nb = NaiveBayes()\n",
    "nb.fit(x_train_naive, y_train_naive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93375\n"
     ]
    }
   ],
   "source": [
    "nb.evaluate(x_train_naive, y_train_naive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.665\n"
     ]
    }
   ],
   "source": [
    "nb.evaluate(x_test_naive, y_test_naive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gaussian Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "naive_bayes = nb.GaussianNB()\n",
    "naive_bayes = naive_bayes.fit(x_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_predict = naive_bayes.predict(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train status = #838 True, #62 False, %93.11111111111111 Accuracy\n"
     ]
    }
   ],
   "source": [
    "cm_train = confusion_matrix(y_train,y_train_predict)\n",
    "t_train,f_train,acc_train = accuracy_on_cm(cm_train)\n",
    "print('Train status = #{} True, #{} False, %{} Accuracy'.format(t_train,f_train,acc_train*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Guassian Test status = #51 True, #49 False, %51.0 Accuracy\n"
     ]
    }
   ],
   "source": [
    "y_test_predict = naive_bayes.predict(x_test)\n",
    "cm_test = confusion_matrix(y_test,y_test_predict)\n",
    "t_test,f_test,acc_test = accuracy_on_cm(cm_test)\n",
    "print('Guassian Test status = #{} True, #{} False, %{} Accuracy'.format(t_test,f_test,acc_test*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multinomial Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "naive_bayes = nb.MultinomialNB()\n",
    "naive_bayes = naive_bayes.fit(x_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_predict = naive_bayes.predict(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multinomial Train status = #772 True, #128 False, %85.77777777777777 Accuracy\n"
     ]
    }
   ],
   "source": [
    "cm_train = confusion_matrix(y_train,y_train_predict)\n",
    "t_train,f_train,acc_train = accuracy_on_cm(cm_train)\n",
    "print('Multinomial Train status = #{} True, #{} False, %{} Accuracy'.format(t_train,f_train,acc_train*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multinomial Test status = #70 True, #30 False, %70.0 Accuracy\n"
     ]
    }
   ],
   "source": [
    "y_test_predict = naive_bayes.predict(x_test)\n",
    "cm_test = confusion_matrix(y_test,y_test_predict)\n",
    "t_test,f_test,acc_test = accuracy_on_cm(cm_test)\n",
    "print('Multinomial Test status = #{} True, #{} False, %{} Accuracy'.format(t_test,f_test,acc_test*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bernoulli Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "naive_bayes = nb.BernoulliNB()\n",
    "naive_bayes = naive_bayes.fit(x_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bernoulli Train status = #635 True, #265 False, %70.55555555555556 Accuracy\n",
      "Bernoulli Test status = #47 True, #53 False, %47.0 Accuracy\n"
     ]
    }
   ],
   "source": [
    "y_train_predict = naive_bayes.predict(x_train)\n",
    "cm_train = confusion_matrix(y_train,y_train_predict)\n",
    "t_train,f_train,acc_train = accuracy_on_cm(cm_train)\n",
    "print('Bernoulli Train status = #{} True, #{} False, %{} Accuracy'.format(t_train,f_train,acc_train*100))\n",
    "\n",
    "y_test_predict = naive_bayes.predict(x_test)\n",
    "cm_test = confusion_matrix(y_test,y_test_predict)\n",
    "t_test,f_test,acc_test = accuracy_on_cm(cm_test)\n",
    "print('Bernoulli Test status = #{} True, #{} False, %{} Accuracy'.format(t_test,f_test,acc_test*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Complement Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "naive_bayes = nb.ComplementNB()\n",
    "naive_bayes = naive_bayes.fit(x_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bernoulli Train status = #745 True, #155 False, %82.77777777777777 Accuracy\n",
      "Bernoulli Test status = #73 True, #27 False, %73.0 Accuracy\n"
     ]
    }
   ],
   "source": [
    "y_train_predict = naive_bayes.predict(x_train)\n",
    "cm_train = confusion_matrix(y_train,y_train_predict)\n",
    "t_train,f_train,acc_train = accuracy_on_cm(cm_train)\n",
    "print('Complement Train status = #{} True, #{} False, %{} Accuracy'.format(t_train,f_train,acc_train*100))\n",
    "\n",
    "y_test_predict = naive_bayes.predict(x_test)\n",
    "cm_test = confusion_matrix(y_test,y_test_predict)\n",
    "t_test,f_test,acc_test = accuracy_on_cm(cm_test)\n",
    "print('Complement Test status = #{} True, #{} False, %{} Accuracy'.format(t_test,f_test,acc_test*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kernel Newton-CG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "logistic_regression = l.LogisticRegression(random_state=85, solver='newton-cg', multi_class='auto')\n",
    "logistic_regression = logistic_regression.fit(x_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bernoulli Train status = #865 True, #35 False, %96.11111111111111 Accuracy\n",
      "Bernoulli Test status = #62 True, #38 False, %62.0 Accuracy\n"
     ]
    }
   ],
   "source": [
    "y_train_predict = logistic_regression.predict(x_train)\n",
    "cm_train = confusion_matrix(y_train,y_train_predict)\n",
    "t_train,f_train,acc_train = accuracy_on_cm(cm_train)\n",
    "print('Logistic Regression Train status = #{} True, #{} False, %{} Accuracy'.format(t_train,f_train,acc_train*100))\n",
    "\n",
    "y_test_predict = logistic_regression.predict(x_test)\n",
    "cm_test = confusion_matrix(y_test,y_test_predict)\n",
    "t_test,f_test,acc_test = accuracy_on_cm(cm_test)\n",
    "print('Logistic Regression Test status = #{} True, #{} False, %{} Accuracy'.format(t_test,f_test,acc_test*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scratch Model\n",
    "In this section we implement Logistic Regression from scratch using numpy library.\n",
    "\n",
    "We will explain our structure in each section provided below:\n",
    "1. Constants\n",
    "1. Cost Function\n",
    "2. Gradient Function\n",
    "3. Learning Parameters using `minimize` optimizer\n",
    "4. Prediction\n",
    "5. Evaluating Model\n",
    "6. Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "m,n = x_train.shape\n",
    "\n",
    "# define theta as zero\n",
    "theta = np.zeros(n)\n",
    "\n",
    "# define hyperparameter λ\n",
    "lambda_ = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Cost Function\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def lr_hypothesis(x,theta):\n",
    "    return np.dot(x,theta)\n",
    "\n",
    "def compute_cost(theta, x, y, lambda_, m, n):\n",
    "    theta = theta.reshape(n,1)\n",
    "    x = x.reshape(m,n)\n",
    "    y = y.reshape(m,1)\n",
    "    infunc1 = -y*(np.log(sigmoid(lr_hypothesis(x,theta)))) - (1-y)*(np.log(1 - sigmoid(lr_hypothesis(x,theta))))\n",
    "    infunc2 = lambda_*np.sum(theta[1:]**2)/(2*m)\n",
    "    j = np.sum(infunc1)/m+infunc2\n",
    "    return j"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Gradient Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "zzz = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gradient[0] correspond to gradient for theta(0)\n",
    "# gradient[1:] correspond to gradient for theta(j) j>0\n",
    "def compute_gradient(theta, x, y, lambda_, m, n):\n",
    "    gradient = np.zeros(n).reshape(n,) # 5,\n",
    "    theta = theta.reshape(n,1) # 5,1\n",
    "    x = x.reshape(m,n)\n",
    "    y = y.reshape(m,1)\n",
    "    infunc1 = sigmoid(lr_hypothesis(x,theta))-y # 10,1\n",
    "    gradient_ = np.dot(x.T,infunc1)/m # shape=(n,1)\n",
    "    gradient_ = gradient_.reshape(n,) # this line not working at all. shape=(1,n) !!!!!!!\n",
    "    #gradient[0] = gradient_[0]    \n",
    "    gradient[1:] = gradient_[1:]+(lambda_*theta[1:,]/m).reshape(n-1,) # theta(j) ; j>0\n",
    "    return gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "if lambda = 3 =======>\n",
      " cost = 2.5348193961097443\n",
      " ,\n",
      " gradients = \n",
      "[ 0.         -0.54855841  0.72472227  1.39800296]\n"
     ]
    }
   ],
   "source": [
    "theta_test = np.array([-2,-1,1,2])\n",
    "\n",
    "x_test = np.append(np.ones(5),np.arange(0.1,1.6,0.1)).reshape(5,4, order='F')\n",
    "\n",
    "y_test = np.array([1,0,1,0,1]).reshape(-1,1)\n",
    "\n",
    "m,n= x_test.shape\n",
    "\n",
    "cost_temp = compute_cost(theta=theta_test,x=x_test,y=y_test,lambda_=3,m=5, n=4)\n",
    "gradient_temp = compute_gradient(theta=theta_test,x=x_test,y=y_test,lambda_=3, m=5, n=4)\n",
    "\n",
    "print('if lambda = 3 =======>\\n cost = {}\\n ,\\n gradients = \\n{}'\n",
    "      .format(cost_temp,gradient_temp))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. Learning Parameters Using `fmin_cg` Optimizer\n",
    "**Scipy's fmin_cg** is an optimization solver that finds the **minimum of an unconstrained** function. For regularized logistic regression, you want to **optimize the cost function J(θ) with parameters θ**. Concretely, you are going to use minimize to find the best parameters θ for the regularized logistic regression cost function, given a fixed dataset (of x and y values). You will pass to minimize the following inputs:\n",
    "1. The initial values of the parameters we are trying to optimize.\n",
    "2. A function that, when given the training set and a particular θ, computes the regularized logistic regression cost with respect to θ for the dataset (x, y) ======> compute_cost\n",
    "3. A function that, when given the training set and a particular θ, computes the regularized logistic regression gradient with respect to θ for the dataset (x, y) ======> compute_gradient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5. Evaluting Model\n",
    "We need to calculate **probabilities and related predictions** and then compare predicted value to real one to get accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [],
   "source": [
    "m,n = x_train.shape\n",
    "lambda_=0.1\n",
    "theta = np.zeros(shape=(n,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library\n",
    "\n",
    "def one_vs_all(theta,x,y,num_labels,lambda_, m, n):\n",
    "    all_theta = np.zeros(shape=(num_labels,n))\n",
    "    \n",
    "    \n",
    "    for i in range(0,num_labels):\n",
    "        all_theta[i] = opt.fmin_cg(f=compute_cost, fprime=compute_gradient, \n",
    "                                   x0 = theta, args=(x,(y==i)*1, lambda_, m, n), full_output=True)\n",
    "        #optimized = opt.minimize(compute_cost, theta, args=(x,(y==i)*1,lambda_),\n",
    "        #                     method=None, jac= True)\n",
    "        #all_theta[c] = optimized.X\n",
    "    return all_theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "operands could not be broadcast together with shapes (0,10865) (10864,) ",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-279-b8efbaa895fa>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m all_theta = one_vs_all(theta.flatten() ,x_train.flatten() , y_train, len(valid_tags), lambda_,\n\u001b[1;32m----> 2\u001b[1;33m                        m=x_train.shape[0], n=x_train.shape[1])\n\u001b[0m",
      "\u001b[1;32m<ipython-input-278-1e0de5359c41>\u001b[0m in \u001b[0;36mone_vs_all\u001b[1;34m(theta, x, y, num_labels, lambda_, m, n)\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mnum_labels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m         all_theta[i] = opt.fmin_cg(f=compute_cost, fprime=compute_gradient, \n\u001b[1;32m---> 10\u001b[1;33m                                    x0 = theta, args=(x,(y==i)*1, lambda_, m, n), full_output=True)\n\u001b[0m\u001b[0;32m     11\u001b[0m         \u001b[1;31m#optimized = opt.minimize(compute_cost, theta, args=(x,(y==i)*1,lambda_),\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m         \u001b[1;31m#                     method=None, jac= True)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\scipy\\optimize\\optimize.py\u001b[0m in \u001b[0;36mfmin_cg\u001b[1;34m(f, x0, fprime, args, gtol, norm, epsilon, maxiter, full_output, disp, retall, callback)\u001b[0m\n\u001b[0;32m   1212\u001b[0m             'return_all': retall}\n\u001b[0;32m   1213\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1214\u001b[1;33m     \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_minimize_cg\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfprime\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mopts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1215\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1216\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfull_output\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\scipy\\optimize\\optimize.py\u001b[0m in \u001b[0;36m_minimize_cg\u001b[1;34m(fun, x0, args, jac, callback, gtol, norm, eps, maxiter, disp, return_all, **unknown_options)\u001b[0m\n\u001b[0;32m   1263\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1264\u001b[0m         \u001b[0mgrad_calls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmyfprime\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mwrap_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfprime\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1265\u001b[1;33m     \u001b[0mgfk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmyfprime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1266\u001b[0m     \u001b[0mk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1267\u001b[0m     \u001b[0mxk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\scipy\\optimize\\optimize.py\u001b[0m in \u001b[0;36mfunction_wrapper\u001b[1;34m(*wrapper_args)\u001b[0m\n\u001b[0;32m    291\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mwrapper_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    292\u001b[0m         \u001b[0mncalls\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 293\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwrapper_args\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    294\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    295\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mncalls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-272-9921e4bd431e>\u001b[0m in \u001b[0;36mcompute_gradient\u001b[1;34m(theta, x, y, lambda_, m, n)\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[0mgradient_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgradient_\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[1;31m#gradient[0] = gradient_[0]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[0mgradient\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgradient_\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlambda_\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mtheta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mm\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# theta(j) ; j>0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: operands could not be broadcast together with shapes (0,10865) (10864,) "
     ]
    }
   ],
   "source": [
    "all_theta = one_vs_all(theta.flatten() ,x_train.flatten() , y_train, len(valid_tags), lambda_,\n",
    "                       m=x_train.shape[0], n=x_train.shape[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kernel SAGA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'x_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-40-1b54d5046344>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mlogistic_regression\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mLogisticRegression\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m85\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msolver\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'saga'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmulti_class\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'auto'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m \u001b[0mlogistic_regression\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlogistic_regression\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'x_train' is not defined"
     ]
    }
   ],
   "source": [
    "logistic_regression = l.LogisticRegression(random_state=85, solver='saga', multi_class='auto')\n",
    "logistic_regression = logistic_regression.fit(x_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bernoulli Train status = #649 True, #251 False, %72.11111111111111 Accuracy\n",
      "Bernoulli Test status = #63 True, #37 False, %63.0 Accuracy\n"
     ]
    }
   ],
   "source": [
    "y_train_predict = logistic_regression.predict(x_train)\n",
    "cm_train = confusion_matrix(y_train,y_train_predict)\n",
    "t_train,f_train,acc_train = accuracy_on_cm(cm_train)\n",
    "print('Saga Train status = #{} True, #{} False, %{} Accuracy'.format(t_train,f_train,acc_train*100))\n",
    "\n",
    "y_test_predict = logistic_regression.predict(x_test)\n",
    "cm_test = confusion_matrix(y_test,y_test_predict)\n",
    "t_test,f_test,acc_test = accuracy_on_cm(cm_test)\n",
    "print('Saga Test status = #{} True, #{} False, %{} Accuracy'.format(t_test,f_test,acc_test*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "svc = svm.SVC(kernel='linear', random_state=85, gamma='auto')\n",
    "svc = svc.fit(x_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bernoulli Train status = #866 True, #34 False, %96.22222222222221 Accuracy\n",
      "Bernoulli Test status = #53 True, #47 False, %53.0 Accuracy\n"
     ]
    }
   ],
   "source": [
    "y_train_predict = svc.predict(x_train)\n",
    "cm_train = confusion_matrix(y_train,y_train_predict)\n",
    "t_train,f_train,acc_train = accuracy_on_cm(cm_train)\n",
    "print('SVM Train status = #{} True, #{} False, %{} Accuracy'.format(t_train,f_train,acc_train*100))\n",
    "\n",
    "y_test_predict = svc.predict(x_test)\n",
    "cm_test = confusion_matrix(y_test,y_test_predict)\n",
    "t_test,f_test,acc_test = accuracy_on_cm(cm_test)\n",
    "print('SVM Test status = #{} True, #{} False, %{} Accuracy'.format(t_test,f_test,acc_test*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple ANN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN + RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}