Zs Data Science Challenge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in c:\\users\\user\\anaconda3\\lib\\site-packages (3.4.5)\n",
      "Requirement already satisfied: six in c:\\users\\user\\anaconda3\\lib\\site-packages (from nltk) (1.14.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install nltk\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gensim in c:\\users\\user\\anaconda3\\lib\\site-packages (4.1.2)\n",
      "Requirement already satisfied: Cython==0.29.23 in c:\\users\\user\\anaconda3\\lib\\site-packages (from gensim) (0.29.23)\n",
      "Requirement already satisfied: scipy>=0.18.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from gensim) (1.4.1)\n",
      "Requirement already satisfied: numpy>=1.17.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from gensim) (1.18.1)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from gensim) (5.2.1)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\user\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\stopwords.zip.\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\user\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\user\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\user\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "#for text pre-processing\n",
    "import re, string\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import SnowballStemmer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "#for model-building\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
    "# bag of words\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for word embedding\n",
    "import gensim\n",
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COMMENT_ID</th>\n",
       "      <th>AUTHOR</th>\n",
       "      <th>DATE</th>\n",
       "      <th>CONTENT</th>\n",
       "      <th>CLASS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LZQPQhLyRh80UYxNuaDWhIGQYNQ96IuCg-AYWqNPjpU</td>\n",
       "      <td>Julius NM</td>\n",
       "      <td>2013-11-07T06:20:48</td>\n",
       "      <td>Huh, anyway check out this you[tube] channel: ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>z13jhp0bxqncu512g22wvzkasxmvvzjaz04</td>\n",
       "      <td>ElNino Melendez</td>\n",
       "      <td>2013-11-09T08:28:43</td>\n",
       "      <td>me shaking my sexy ass on my channel enjoy ^_^ ﻿</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>z13fwbwp1oujthgqj04chlngpvzmtt3r3dw</td>\n",
       "      <td>GsMega</td>\n",
       "      <td>2013-11-10T16:05:38</td>\n",
       "      <td>watch?v=vtaRGgvGtWQ   Check this out .﻿</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>z13lfzdo5vmdi1cm123te5uz2mqig1brz04</td>\n",
       "      <td>ferleck ferles</td>\n",
       "      <td>2013-11-27T21:39:24</td>\n",
       "      <td>Subscribe to my channel ﻿</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>z12avveb4xqiirsix04chxviiljryduwxg0</td>\n",
       "      <td>BeBe Burkey</td>\n",
       "      <td>2013-11-28T16:30:13</td>\n",
       "      <td>and u should.d check my channel and tell me wh...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1152</th>\n",
       "      <td>_2viQ_Qnc6_RKHVetk9kLzx8ZC62_J7y73FWFSBTe8Q</td>\n",
       "      <td>ThirdDegr3e</td>\n",
       "      <td>2013-07-13T20:47:40.793000</td>\n",
       "      <td>**CHECK OUT MY NEW MIXTAPE**** **CHECK OUT MY ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1153</th>\n",
       "      <td>_2viQ_Qnc68TufyXKiTwky80ewSPbhRiD5XFHrJH9lg</td>\n",
       "      <td>Ysobel Schofield</td>\n",
       "      <td>2013-07-13T20:17:25.181000</td>\n",
       "      <td>Waka waka she rules</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1154</th>\n",
       "      <td>_2viQ_Qnc6-pY-1yR6K2FhmC5i48-WuNx5CumlHLDAI</td>\n",
       "      <td>Sabina Pearson-Smith</td>\n",
       "      <td>2013-07-13T13:14:30.021000</td>\n",
       "      <td>I love this song for two reasons: 1.it is abou...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1155</th>\n",
       "      <td>_2viQ_Qnc6_k_n_Bse9zVhJP8tJReZpo8uM2uZfnzDs</td>\n",
       "      <td>jeffrey jules</td>\n",
       "      <td>2013-07-13T12:09:31.188000</td>\n",
       "      <td>wow</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1156</th>\n",
       "      <td>_2viQ_Qnc6_yBt8UGMWyg3vh0PulTqcqyQtdE7d4Fl0</td>\n",
       "      <td>Aishlin Maciel</td>\n",
       "      <td>2013-07-13T11:17:52.308000</td>\n",
       "      <td>Shakira u are so wiredo</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1157 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       COMMENT_ID                AUTHOR  \\\n",
       "0     LZQPQhLyRh80UYxNuaDWhIGQYNQ96IuCg-AYWqNPjpU             Julius NM   \n",
       "1             z13jhp0bxqncu512g22wvzkasxmvvzjaz04       ElNino Melendez   \n",
       "2             z13fwbwp1oujthgqj04chlngpvzmtt3r3dw                GsMega   \n",
       "3             z13lfzdo5vmdi1cm123te5uz2mqig1brz04        ferleck ferles   \n",
       "4             z12avveb4xqiirsix04chxviiljryduwxg0           BeBe Burkey   \n",
       "...                                           ...                   ...   \n",
       "1152  _2viQ_Qnc6_RKHVetk9kLzx8ZC62_J7y73FWFSBTe8Q           ThirdDegr3e   \n",
       "1153  _2viQ_Qnc68TufyXKiTwky80ewSPbhRiD5XFHrJH9lg      Ysobel Schofield   \n",
       "1154  _2viQ_Qnc6-pY-1yR6K2FhmC5i48-WuNx5CumlHLDAI  Sabina Pearson-Smith   \n",
       "1155  _2viQ_Qnc6_k_n_Bse9zVhJP8tJReZpo8uM2uZfnzDs         jeffrey jules   \n",
       "1156  _2viQ_Qnc6_yBt8UGMWyg3vh0PulTqcqyQtdE7d4Fl0        Aishlin Maciel   \n",
       "\n",
       "                            DATE  \\\n",
       "0            2013-11-07T06:20:48   \n",
       "1            2013-11-09T08:28:43   \n",
       "2            2013-11-10T16:05:38   \n",
       "3            2013-11-27T21:39:24   \n",
       "4            2013-11-28T16:30:13   \n",
       "...                          ...   \n",
       "1152  2013-07-13T20:47:40.793000   \n",
       "1153  2013-07-13T20:17:25.181000   \n",
       "1154  2013-07-13T13:14:30.021000   \n",
       "1155  2013-07-13T12:09:31.188000   \n",
       "1156  2013-07-13T11:17:52.308000   \n",
       "\n",
       "                                                CONTENT  CLASS  \n",
       "0     Huh, anyway check out this you[tube] channel: ...      1  \n",
       "1      me shaking my sexy ass on my channel enjoy ^_^ ﻿      1  \n",
       "2               watch?v=vtaRGgvGtWQ   Check this out .﻿      1  \n",
       "3                             Subscribe to my channel ﻿      1  \n",
       "4     and u should.d check my channel and tell me wh...      1  \n",
       "...                                                 ...    ...  \n",
       "1152  **CHECK OUT MY NEW MIXTAPE**** **CHECK OUT MY ...      1  \n",
       "1153                                Waka waka she rules      0  \n",
       "1154  I love this song for two reasons: 1.it is abou...      0  \n",
       "1155                                                wow      0  \n",
       "1156                            Shakira u are so wiredo      0  \n",
       "\n",
       "[1157 rows x 5 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#LOADING DATA\n",
    "df_train=pd.read_csv(\"train.csv\",sep=\",\")\n",
    "df_test=pd.read_csv(\"test.csv\",sep=\",\")\n",
    "df_train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>COMMENT_ID</th>\n",
       "      <th>AUTHOR</th>\n",
       "      <th>DATE</th>\n",
       "      <th>CONTENT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>LZQPQhLyRh_C2cTtd9MvFRJedxydaVW-2sNg5Diuo4A</td>\n",
       "      <td>adam riyati</td>\n",
       "      <td>2013-11-07T12:37:15</td>\n",
       "      <td>Hey guys check out my new channel and our firs...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>LZQPQhLyRh9MSZYnf8djyk0gEF9BHDPYrrK-qCczIY8</td>\n",
       "      <td>Evgeny Murashkin</td>\n",
       "      <td>2013-11-08T17:34:21</td>\n",
       "      <td>just for test I have to say murdev.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>LZQPQhLyRh9-wNRtlZDM90f1k0BrdVdJyN_YsaSwfxc</td>\n",
       "      <td>Jason Haddad</td>\n",
       "      <td>2013-11-26T02:55:11</td>\n",
       "      <td>Hey, check out my new website!! This site is a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>z122wfnzgt30fhubn04cdn3xfx2mxzngsl40k</td>\n",
       "      <td>Bob Kanowski</td>\n",
       "      <td>2013-11-28T12:33:27</td>\n",
       "      <td>i turned it on mute as soon is i came on i jus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>z13ttt1jcraqexk2o234ghbgzxymz1zzi04</td>\n",
       "      <td>Cony</td>\n",
       "      <td>2013-11-28T16:01:47</td>\n",
       "      <td>You should check my channel for Funny VIDEOS!!﻿</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>794</th>\n",
       "      <td>794</td>\n",
       "      <td>_2viQ_Qnc6-jk58CPwBnqfbM6oByJH5oPvCtKecLQyo</td>\n",
       "      <td>Shadrach Grentz</td>\n",
       "      <td>2013-07-14T03:11:20.243000</td>\n",
       "      <td>Hey Music Fans I really appreciate all of you ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>795</th>\n",
       "      <td>795</td>\n",
       "      <td>_2viQ_Qnc689m-WiwOwvrQU7LvkLAgspnfXL8ovE0ME</td>\n",
       "      <td>TheHotChocolate</td>\n",
       "      <td>2013-07-13T18:26:37.017000</td>\n",
       "      <td>she is sooooo beautiful!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796</th>\n",
       "      <td>796</td>\n",
       "      <td>_2viQ_Qnc6_1Hq9MGlefkBIszt9rYD3S_CozADvMhQ4</td>\n",
       "      <td>Dinova Sharon</td>\n",
       "      <td>2013-07-13T14:44:00.700000</td>\n",
       "      <td>well done shakira</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>797</td>\n",
       "      <td>_2viQ_Qnc6-bMSjqyL1NKj57ROicCSJV5SwTrw-RFFA</td>\n",
       "      <td>Katie Mettam</td>\n",
       "      <td>2013-07-13T13:27:39.441000</td>\n",
       "      <td>I love this song because we sing it at Camp al...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>798</td>\n",
       "      <td>_2viQ_Qnc685RPw1aSa1tfrIuHXRvAQ2rPT9R06KTqA</td>\n",
       "      <td>Latin Bosch</td>\n",
       "      <td>2013-07-12T22:33:27.916000</td>\n",
       "      <td>Shakira is the best dancer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>799 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ID                                   COMMENT_ID            AUTHOR  \\\n",
       "0      0  LZQPQhLyRh_C2cTtd9MvFRJedxydaVW-2sNg5Diuo4A       adam riyati   \n",
       "1      1  LZQPQhLyRh9MSZYnf8djyk0gEF9BHDPYrrK-qCczIY8  Evgeny Murashkin   \n",
       "2      2  LZQPQhLyRh9-wNRtlZDM90f1k0BrdVdJyN_YsaSwfxc      Jason Haddad   \n",
       "3      3        z122wfnzgt30fhubn04cdn3xfx2mxzngsl40k      Bob Kanowski   \n",
       "4      4          z13ttt1jcraqexk2o234ghbgzxymz1zzi04              Cony   \n",
       "..   ...                                          ...               ...   \n",
       "794  794  _2viQ_Qnc6-jk58CPwBnqfbM6oByJH5oPvCtKecLQyo   Shadrach Grentz   \n",
       "795  795  _2viQ_Qnc689m-WiwOwvrQU7LvkLAgspnfXL8ovE0ME   TheHotChocolate   \n",
       "796  796  _2viQ_Qnc6_1Hq9MGlefkBIszt9rYD3S_CozADvMhQ4     Dinova Sharon   \n",
       "797  797  _2viQ_Qnc6-bMSjqyL1NKj57ROicCSJV5SwTrw-RFFA      Katie Mettam   \n",
       "798  798  _2viQ_Qnc685RPw1aSa1tfrIuHXRvAQ2rPT9R06KTqA       Latin Bosch   \n",
       "\n",
       "                           DATE  \\\n",
       "0           2013-11-07T12:37:15   \n",
       "1           2013-11-08T17:34:21   \n",
       "2           2013-11-26T02:55:11   \n",
       "3           2013-11-28T12:33:27   \n",
       "4           2013-11-28T16:01:47   \n",
       "..                          ...   \n",
       "794  2013-07-14T03:11:20.243000   \n",
       "795  2013-07-13T18:26:37.017000   \n",
       "796  2013-07-13T14:44:00.700000   \n",
       "797  2013-07-13T13:27:39.441000   \n",
       "798  2013-07-12T22:33:27.916000   \n",
       "\n",
       "                                               CONTENT  \n",
       "0    Hey guys check out my new channel and our firs...  \n",
       "1               just for test I have to say murdev.com  \n",
       "2    Hey, check out my new website!! This site is a...  \n",
       "3    i turned it on mute as soon is i came on i jus...  \n",
       "4      You should check my channel for Funny VIDEOS!!﻿  \n",
       "..                                                 ...  \n",
       "794  Hey Music Fans I really appreciate all of you ...  \n",
       "795                           she is sooooo beautiful!  \n",
       "796                                  well done shakira  \n",
       "797  I love this song because we sing it at Camp al...  \n",
       "798                         Shakira is the best dancer  \n",
       "\n",
       "[799 rows x 5 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    586\n",
      "0    571\n",
      "Name: CLASS, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1e2dc773a48>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPBUlEQVR4nO3df4xlZX3H8fenrIix6vJjQLK7dWnc+KO1Ck6QlvSHbtsANS5tJZGYsiEbt3/QRkNTpf1D0yY2mjalkjQ0G1GXVq2E1rJRqsUFY5sWylARpKsyJcpOF9lRftUSa6nf/jHPPMzuXnYWlnPvsPN+JTfnOd/z3LvfSTbzyXnOuWdSVUiSBPAjk25AkrRyGAqSpM5QkCR1hoIkqTMUJEndmkk3cDROOeWU2rhx46TbkKTnlDvuuOM7VTU16thzOhQ2btzIzMzMpNuQpOeUJN96qmMuH0mSukFDIcnaJNcn+VqSPUl+OslJSW5Kcm/bntjmJslVSWaT3JXkrCF7kyQdaugzhQ8Bn6uqVwKvBfYAVwC7q2oTsLvtA5wPbGqv7cDVA/cmSTrIYKGQ5MXAzwHXAFTVD6rqEWALsLNN2wlc2MZbgGtrwa3A2iSnD9WfJOlQQ54p/DgwD3w0yZeTfDjJC4HTquoBgLY9tc1fB+xd8v65VjtAku1JZpLMzM/PD9i+JK0+Q4bCGuAs4OqqOhP4b55cKholI2qHPK2vqnZU1XRVTU9NjbyjSpL0DA0ZCnPAXFXd1vavZyEkHlxcFmrb/Uvmb1jy/vXAvgH7kyQdZLBQqKpvA3uTvKKVNgP/DuwCtrbaVuCGNt4FXNLuQjoHeHRxmUmSNB5Df3ntt4GPJzkeuA+4lIUgui7JNuB+4KI290bgAmAWeLzNlSSN0aChUFV3AtMjDm0eMbeAy4bsR3ouuf8PXzPpFrQC/dh77x708/1GsySpMxQkSZ2hIEnqDAVJUvecfnT2s+H1v3vtpFvQCnTHH18y6RakifBMQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJ3aChkOSbSe5OcmeSmVY7KclNSe5t2xNbPUmuSjKb5K4kZw3ZmyTpUOM4U3hjVb2uqqbb/hXA7qraBOxu+wDnA5vaaztw9Rh6kyQtMYnloy3AzjbeCVy4pH5tLbgVWJvk9An0J0mr1tChUMA/JLkjyfZWO62qHgBo21NbfR2wd8l751rtAEm2J5lJMjM/Pz9g65K0+qwZ+PPPrap9SU4FbkrytcPMzYhaHVKo2gHsAJienj7kuCTpmRv0TKGq9rXtfuDTwNnAg4vLQm27v02fAzYseft6YN+Q/UmSDjRYKCR5YZIXLY6BXwa+CuwCtrZpW4Eb2ngXcEm7C+kc4NHFZSZJ0ngMuXx0GvDpJIv/zieq6nNJbgeuS7INuB+4qM2/EbgAmAUeBy4dsDdJ0giDhUJV3Qe8dkT9u8DmEfUCLhuqH0nS8vxGsySpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSd3goZDkuCRfTvKZtn9GktuS3JvkU0mOb/Xnt/3Zdnzj0L1Jkg40jjOFdwJ7lux/ELiyqjYBDwPbWn0b8HBVvRy4ss2TJI3RoKGQZD3wK8CH236ANwHXtyk7gQvbeEvbpx3f3OZLksZk6DOFPwPeDfyw7Z8MPFJVT7T9OWBdG68D9gK044+2+QdIsj3JTJKZ+fn5IXuXpFVnsFBI8mZgf1XdsbQ8YmodwbEnC1U7qmq6qqanpqaehU4lSYvWDPjZ5wJvSXIBcALwYhbOHNYmWdPOBtYD+9r8OWADMJdkDfAS4KEB+5MkHWSwM4Wq+r2qWl9VG4G3ATdX1duBW4C3tmlbgRvaeFfbpx2/uaoOOVOQJA1nEt9TeA9weZJZFq4ZXNPq1wAnt/rlwBUT6E2SVrUhl4+6qvoi8MU2vg84e8Sc7wMXjaMfSdJofqNZktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeqeVigkeV6SM5OcOlRDkqTJOWwoJPmLJD/Rxi8BvgJcC3w5ycVj6E+SNEbLnSn8bFXd08aXAt+oqtcArwfePWhnkqSxWy4UfrBk/EvA3wFU1bcH60iSNDHLhcIjSd6c5EzgXOBzAEnWAC8YujlJ0ngt9zeafxO4Cngp8K4lZwibgc8O2ZgkafwOGwpV9Q3gvBH1zwOfH6opSdJkLHf30TuSbGrjJPlokseS3NWWlCRJx5Dlrim8E/hmG18M/BRwBnA5C8tKkqRjyHKh8ERV/W8bvxm4tqq+W1VfAF44bGuSpHFbLhR+mOT0JCewcHH5C0uOefeRJB1jlrv76L3ADHAcsGvxi2xJfh64b+DeJEljttzdR59J8jLgRVX18JJDtwNvG7QzSdLYLftAvKp6YjEQ2h1Ib2LhIvPs4d6X5IQk/5rkK0nuSfIHrX5GktuS3JvkU0mOb/Xnt/3Zdnzj0f5wkqSn54iekprkDUk+BHwL2AX8I/DKZd72P8Cbquq1wOuA85KcA3wQuLKqNgEPA9va/G3Aw1X1cuDKNk+SNEbLfU/h/UnuBf4IuBs4E5ivqp0HLScdohZ8r+0+r70KeBNwfavvBC5s4y1tn3Z8c5I8zZ9HknQUljtT2A48CFwN/FVVfZeFX+xHJMlxSe4E9gM3Af8BPFJVT7Qpc8C6Nl4H7IWFJSvgUeDkEZ+5PclMkpn5+fkjbUWSdASWC4WXAu8H3gLMJvlL4AXtgXjLqqr/q6rXAeuBs4FXjZrWtqPOCg4JoKraUVXTVTU9NTV1JG1Iko7QYUOh/VL/+6q6BHg5cAPwz8B/JvnEkf4jVfUI8EXgHGDtklBZD+xr4zlgA/SnsL4EeOjIfxRJ0tE64j/HWVXfr6rrq+rXgU3AHYebn2Qqydo2fgHwi8Ae4BbgrW3aVhaCBhYuYG9t47cCN1fVES9VSZKO3tP6G82LquoxFp6LdDinA7ckuYuF7zXcVFWfAd4DXJ5kloVrBte0+dcAJ7f65cAVz6Q3SdIzd0TXBp7CYe8Mqqq7WLhb6eD6fSxcXzi4/n3goqPoR5J0lJ7RmULj0o4kHWMOe6aQ5G5G//IPcNogHUmSJma55aNfY+GX/96D6i/jybuGJEnHiOWWj64EHquqby19AY+3Y5KkY8hyobCxXTA+QFXNABsH6UiSNDHLhcIJhznmH9mRpGPMcqFwe5J3HFxMso1lvrwmSXruWe5C87uATyd5O0+GwDRwPPCrQzYmSRq/5f7y2oPAzyR5I/CTrfzZqrp58M4kSWN3pE87vYWFZxZJko5hR/ONZknSMcZQkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1A0WCkk2JLklyZ4k9yR5Z6uflOSmJPe27YmtniRXJZlNcleSs4bqTZI02pBnCk8Av1NVrwLOAS5L8mrgCmB3VW0Cdrd9gPOBTe21Hbh6wN4kSSMMFgpV9UBV/Vsb/xewB1gHbAF2tmk7gQvbeAtwbS24FVib5PSh+pMkHWos1xSSbATOBG4DTquqB2AhOIBT27R1wN4lb5trtYM/a3uSmSQz8/PzQ7YtSavO4KGQ5EeBvwHeVVWPHW7qiFodUqjaUVXTVTU9NTX1bLUpSWLgUEjyPBYC4eNV9bet/ODislDb7m/1OWDDkrevB/YN2Z8k6UBD3n0U4BpgT1X96ZJDu4CtbbwVuGFJ/ZJ2F9I5wKOLy0ySpPFYM+Bnnwv8BnB3kjtb7feBDwDXJdkG3A9c1I7dCFwAzAKPA5cO2JskaYTBQqGq/onR1wkANo+YX8BlQ/UjSVqe32iWJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpGywUknwkyf4kX11SOynJTUnubdsTWz1Jrkoym+SuJGcN1Zck6akNeabwMeC8g2pXALurahOwu+0DnA9saq/twNUD9iVJegqDhUJVfQl46KDyFmBnG+8ELlxSv7YW3AqsTXL6UL1JkkYb9zWF06rqAYC2PbXV1wF7l8yba7VDJNmeZCbJzPz8/KDNStJqs1IuNGdErUZNrKodVTVdVdNTU1MDtyVJq8u4Q+HBxWWhtt3f6nPAhiXz1gP7xtybJK164w6FXcDWNt4K3LCkfkm7C+kc4NHFZSZJ0visGeqDk3wS+AXglCRzwPuADwDXJdkG3A9c1KbfCFwAzAKPA5cO1Zck6akNFgpVdfFTHNo8Ym4Blw3ViyTpyKyUC82SpBXAUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKlbUaGQ5LwkX08ym+SKSfcjSavNigmFJMcBfw6cD7wauDjJqyfblSStLismFICzgdmquq+qfgD8NbBlwj1J0qqyZtINLLEO2Ltkfw54w8GTkmwHtrfd7yX5+hh6Wy1OAb4z6SZWgvzJ1km3oAP5f3PR+/JsfMrLnurASgqFUT9pHVKo2gHsGL6d1SfJTFVNT7oP6WD+3xyflbR8NAdsWLK/Htg3oV4kaVVaSaFwO7ApyRlJjgfeBuyacE+StKqsmOWjqnoiyW8BnweOAz5SVfdMuK3VxmU5rVT+3xyTVB2ybC9JWqVW0vKRJGnCDAVJUmcoyMeLaMVK8pEk+5N8ddK9rBaGwirn40W0wn0MOG/STawmhoJ8vIhWrKr6EvDQpPtYTQwFjXq8yLoJ9SJpwgwFHdHjRSStDoaCfLyIpM5QkI8XkdQZCqtcVT0BLD5eZA9wnY8X0UqR5JPAvwCvSDKXZNukezrW+ZgLSVLnmYIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKk7v8BeObEwLnJGP8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x=df_train['CLASS'].value_counts()\n",
    "print(x)\n",
    "sns.barplot(x.index,x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1157 entries, 0 to 1156\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   COMMENT_ID  1157 non-null   object\n",
      " 1   AUTHOR      1157 non-null   object\n",
      " 2   DATE        1019 non-null   object\n",
      " 3   CONTENT     1157 non-null   object\n",
      " 4   CLASS       1157 non-null   int64 \n",
      "dtypes: int64(1), object(4)\n",
      "memory usage: 45.3+ KB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 799 entries, 0 to 798\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   ID          799 non-null    int64 \n",
      " 1   COMMENT_ID  799 non-null    object\n",
      " 2   AUTHOR      799 non-null    object\n",
      " 3   DATE        692 non-null    object\n",
      " 4   CONTENT     799 non-null    object\n",
      "dtypes: int64(1), object(4)\n",
      "memory usage: 31.3+ KB\n"
     ]
    }
   ],
   "source": [
    "df_train.info()\n",
    "df_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "COMMENT_ID      0\n",
       "AUTHOR          0\n",
       "DATE          138\n",
       "CONTENT         0\n",
       "CLASS           0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Missing value\n",
    "df_train.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22.12627986348123\n",
      "8.84938704028021\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAEVCAYAAAAigatAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de5hlVX3n//dHmpuANpcGgcY0KM6ITmhJBxlxMgaMAjpCntH8MP6kyWDaTHBGEzMKmlFIZEYzUQxPEhMQ5TIqEqIDwzCJhEuMSQCbiFxEQwsITTd0KxdRIxH8zh97lRyKqq7TXfvUrd+v59nP2Xutdfb+7nOqVn1r31aqCkmSJE3fM2Y7AEmSpIXCxEqSJKknJlaSJEk9MbGSJEnqiYmVJElST0ysJEmSemJiJWmLJDktyf+c7TgkaS4xsZIWgCSnJrliXNkdk5QdP7PRaTqSXJvkLbMdh6ThmFhJC8MXgcOTbAOQ5DnAtsAh48qe39oOLZ0511ckWTTbMUjSeHOus5S0Rb5Ml0gtb8s/B1wDfGNc2Terah1Akpcl+XKSR9rry8ZW1o6SnJHkb4EfAAck2T/JXyd5NMmVwB4D7XdI8j+TfCfJw219e00UaJK72xG2ryV5KMknk+wwUP/aJDe19fxdkp8e9953J7kZ+P5EyVWSFyW5MsmDSR5I8p5Wvn2SjyZZ16aPJtm+1b0iydok70qyIcn6JMclOSbJP7Z1vWdgG6cl+bO2z48muSXJC9p+bUhyb5JXDbR/dpJz23rvS/KBgYT3xCRfSvL77fO4K8nRre4M4N8Af5jke0n+cNM/BpJmm4mVtABU1T8D19MlT7TXvwG+NK7siwBJdgP+D3AWsDvwEeD/JNl9YLVvBlYBuwDfAj4N3EiXUP0usHKg7Urg2cB+bX2/BvzTJkJ+E/Bq4HnAC4DfbnEdAnwCeGtbz58Cl40lQM0bgdcAi6vq8cGVJtkF+CvgL4B96I7QXdWq3wscRpdoHgwcOrbd5jnADsC+wPuAc4D/H/gZuuTmfUkOGGj/74ALgV2BrwB/Sden7gv8Tot9zPnA4y2elwCvAgZP772ULgneA/g94Nwkqar30n2Pb6uqnavqbRN9mJLmDhMraeH4a55Mov4N3R/kvxlX9tdt/jXAHVV1YVU9XlWfAb5OlyyMOa+qbmvJy97AzwL/taoeq6ovAv97oO2P6BKh51fVE1V1Y1V9dxOx/mFV3VtVDwJn0CVLAL8K/GlVXd/Wcz7wGF1CNOas9t6JErfXAvdX1Yer6odV9WhVXd/q3gT8TlVtqKqNwOl0yePgPpxRVT8CLqJLcv6greM24Dbgpwfa/01V/WX7fP4MWAJ8cOD9y5IsbkfujgbeUVXfr6oNwJnA4LVu36qqc6rqCbokbG9gwiN+kuY2Eytp4fgi8PIkuwJLquoO4O+Al7WyF/Pk9VX70B2FGvQtuqMtY+4dmN8HeKiqvj+u/ZgL6Y7YXNROs/1ekm03Eevgur/V1g/wU8A722nAh5M8THcUbJ9J3jvefsA3J6kbv8+D2wX4Tkts4MmjbQ8M1P8TsPPA8vi6b0/w/p3p9mlbYP3APv0psOfA++8fm6mqHwy8V9I8Y2IlLRx/T3c6bhXwtwDtqNG6Vrauqu5qbdfR/cEf9FzgvoHlGphfD+yaZKdx7Wnb+VFVnV5VBwEvoztydMImYt1v3HrWtfl76Y4aLR6YntmOqE0U13j30p1enMj4fR7c7ijdS3fUbY+BfXpWVb1oyPdvan8lzTEmVtIC0U6NrQZ+k+4U4JgvtbLBuwGvAF6Q5JeTLEry/wEHAZdPsu5vtXWfnmS7JC9n4LRhkp9P8q/aBdnfpTut9sRE62pOTrK0Xev1HuCzrfwc4NeSvLTdjbhTkte0a6eGcTnwnCTvaBer75Lkpa3uM8BvJ1mSZA+666hG/hyuqloPfAH4cJJnJXlGkucl+bdDruIB4IApW0maE0yspIXlr+lOMX1poOxvWtlPEquq+g7dUaV3At8B3gW8tqq+vYl1/zLdRdYPAu8HLhioew5wCV1SdXuLY1NJy6fpko072/SBFtdquuus/hB4CFgDnLiJ9TxFVT0K/AJd0nc/cAfw8636A3TJ4c3ALcA/jG13BpwAbAd8jW6/LqG7jmoYfwC8vt0xeNaI4pPUk1R5lFnSzElyN/CWqvqr2Y5FkvrmEStJkqSemFhJkiT1xMRKPzH2BOjZjkMLW1Ut8zSg5pKxJ+/PdhxaGEystgLphid5aNzTq7d6bXiUV852HNJC137XHhh8XEeStyS5dsTbPS/J40n2mbr11iMO7D1SJlYLXJJldE/cLuB1sxqMpK3ZIuDtM7WxlsT9e+ARuqfuSzPCxGrhOwG4DjiPp47tRpLdk1yW5LtJbmDyByuOtX95ukFxH26DzJ7Yyp+d5IIkG5N8K8lvJ3lGqzsxyd8mObO97850g/+e2NaxIcnKgW2cl+SPk/zfNujs3yZ5TroBcx9K8vUkLxlov0+SP2/bvivJfx6oOy3JxS22R5PclmRFq7uQ7gGR/7tt513ZjIGEJW22/wH8VpLFE1Vm6kHBf7f1B48m+UJ7Ftmm/HvgYbpxG8f3fTu2vuahJF+jG65pUlngA3unc2bbziNJbk7y4ik+X02mqpwW8ET3HKBfpxtI9kfAXgN1FwEXAzvRDXdyH/ClSdbzXOBRujHdtqUbF255q7sAuJRusN5lwD8CJ7W6E+kGn/0VYBu65wbdA/wRsD3dYLSPAju39ucB327x7gBcDdxFlyCOvf+a1vYZdIMCv4/uGUEH0D0T6dWt/jTgh8Ax7b3/HbhuYJ/uBl45sPxWuvHvntna/wzwrNn+Dp2c5vs09rsGfA74QCt7C3Btm9+N7vleb6Y7svXGtrx7q7+WbqiiFwA7tuUPTrHNq+gGtN6r9UGHDNR9kO75brvRjQJwK7B2kvXsQjfywDtbn7QL8NJW9zt0/7juSTdW5N8Bv9vqXtG2+77WZ/4qsJHuGW67AC9q/dMBrf1Yf/Xq9hlc0Pq+9w68/66BuP4X3dBIO7Xt3wC8tdWdSNff/2rry/4j3SgDGfg83zKwrle3vnQxEOCFwN6z/XMzX6dZD8BphF8uvLz9cu3Rlr8O/Eab36bV/cuB9v+NyROrU4HPT1C+Dd1wHQcNlL11oMM8kW6w37G6f0V3WnIwwfsOTyZp5wHnDNT9J+D2ce9/uM2/FLhngjg/2eZPA/5qoO4g4J8Glu/mqYnVf2gd40/P9nfn5LSQJp5MrF5Md2puCU9NrN4M3DDuPX8PnNjmrwV+e6Du14G/2MT2ngv8eKBf+Uu6AbXH6u8EjhpYXsXkidUbga9MUvdN4JiB5VcDd7f5V9CNGblNW96l9X0vHWh/I3Bcmz8NuHKg7t8B35vg/YvpksXHgB3HxXlNmz8RWDNQ98z23ucMfJ6DidURdP8QHwY8Y7Z/Xub75KnAhW0l8IV68mnan+bJQ+JL6P4rGj8Y7mQmG9x2D7qjReMHtx0czHf8YLVU1eYMbjtZ258C9slTB+x9D12nM+b+gfkfADskWTTBfsDmDyQsaTNU1a10ww6dMq5qmEHBx/8u7wyQ5E/aKa3vDZxaezPdP2Q3teVPAb888Pu8D9Pv+yaKe14O7F1VV9ONdvBHwANJzk7yrInaamomVgtUkh2BXwL+bZL7k9wP/AZwcJKD6Q5JP87TB8OdzGSD236b7sjX+MFt75ugbd/upTs0Pjhg7y5VdcyQ73/KsAO1+QMJS9p876c7RTWYNA0zKPiEqurXqmrnNv23VnwCcMBA3/cRun8Cj27165l+3zdR3PN2YO+qOquqfobuFOULgP/SX7hbFxOrhes4ukFwDwKWt+mFdNcVnND+C/occFqSZyY5iHEXeI7zKeCVSX4p3aC9uydZ3tZzMXBGugFvf4puwN+RD25Ld03Bd5O8u12Muk2SFyfZ5IWoA54yuG02fyBhSZupqtbQDbr9nweKN2tQ8E1J8q/pEqFDebLvezFPPWJ/MXBqkl2TLKW75GAyC35g7yQ/m27g822B79Nd62Xft4VMrBaulXTXGt1TVfePTXSHe9/UToe9je7Q8P101zZ9crKVVdU9dBeBv5NuEN6bgINb9X+i+2W8k27w308DnxjFTo2L6Qm66xCW013k+W3g48Czh1zFf6frFB9O8lts/kDCkrbM79BddA1s8aDgk1kJXFpVt4zr+/4AeG2S3YDT6U7b3UWXoFw42cpq6xjY+1nAOW0936L7Dn6/94i3Eg7CLEmS1BOPWEmSJPXExEqSJKknJlaSJEk9MbGSJEnqyWQPSpxRe+yxRy1btmy2w5A0g2688cZvV9WS2Y6jD/Zh0tZlU/3XnEisli1bxurVq2c7DEkzKMmmnnY9r9iHSVuXTfVfngqUJEnqiYmVJElST0ysJC14Se5OckuSm5KsbmW7JbkyyR3tdddWniRnJVmT5OYkh8xu9JLmExMrSVuLn6+q5VW1oi2fAlxVVQcCV7Vl6AbqPbBNq4CPzXikkuYtEytJW6tjgfPb/Pl0A5ePlV9QneuAxUmGHYNN0lbOxErS1qCALyS5McmqVrZXVa0HaK97tvJ9gXsH3ru2lT1FklVJVidZvXHjxhGGLmk+mROPW5CkETu8qtYl2RO4MsnXN9E2E5Q9bbT6qjobOBtgxYoVjmYvCfCIlaStQFWta68bgM8DhwIPjJ3ia68bWvO1wH4Db18KrJu5aCXNZyZWkha0JDsl2WVsHngVcCtwGbCyNVsJXNrmLwNOaHcHHgY8MnbKUJKmMj9PBWaiI/U9KY/oSwvMXsDn0/Ubi4BPV9VfJPkycHGSk4B7gDe09lcAxwBrgB8Av9JnMDl9dP1Xvd/+S5pt8zOxkqQhVdWdwMETlH8HOHKC8gJOnoHQJC1AngqUJEnqiYmVJElST0ysJEmSemJiJUmS1BMTK0mSpJ6YWEmSJPXExEqSJKknJlaSJEk9GTqxSrJNkq8kubwt75/k+iR3JPlsku1a+fZteU2rXzaa0CVJkuaWzTli9Xbg9oHlDwFnVtWBwEPASa38JOChqno+cGZrJ0mStOANlVglWQq8Bvh4Ww5wBHBJa3I+cFybP7Yt0+qPbO0lSZIWtGGPWH0UeBfw47a8O/BwVT3eltcC+7b5fYF7AVr9I639UyRZlWR1ktUbN27cwvAlSZLmjikTqySvBTZU1Y2DxRM0rSHqniyoOruqVlTViiVLlgwVrCRJ0ly2aIg2hwOvS3IMsAPwLLojWIuTLGpHpZYC61r7tcB+wNoki4BnAw/2HrkkSdIcM+URq6o6taqWVtUy4Hjg6qp6E3AN8PrWbCVwaZu/rC3T6q+uqqcdsZIkSVpopvMcq3cDv5lkDd01VOe28nOB3Vv5bwKnTC9ESZKk+WGYU4E/UVXXAte2+TuBQydo80PgDT3EJkmSNK/45HVJkqSemFhJkiT1xMRKkiSpJyZWkiRJPTGxkiRJ6omJlSRJUk9MrCRJknpiYiVJktQTEytJkqSemFhJkiT1xMRKkiSpJyZWkiRJPTGxkiRJ6omJlSRJUk9MrCRJknoyZWKVZIckNyT5apLbkpzeys9LcleSm9q0vJUnyVlJ1iS5Ockho94JSZKkuWCYI1aPAUdU1cHAcuCoJIe1uv9SVcvbdFMrOxo4sE2rgI/1HbQkbY4k2yT5SpLL2/L+Sa5PckeSzybZrpVv35bXtPplsxm3pPlnysSqOt9ri9u2qTbxlmOBC9r7rgMWJ9l7+qFK0hZ7O3D7wPKHgDOr6kDgIeCkVn4S8FBVPR84s7WTpKENdY1V+2/vJmADcGVVXd+qzmin+85Msn0r2xe4d+Dta1vZ+HWuSrI6yeqNGzdOYxckaXJJlgKvAT7elgMcAVzSmpwPHNfmj23LtPojW3tJGspQiVVVPVFVy4GlwKFJXgycCvxL4GeB3YB3t+YTdUJPO8JVVWdX1YqqWrFkyZItCl6ShvBR4F3Aj9vy7sDDVfV4Wx785+8n/xi2+kda+6fxn0NJE9msuwKr6mHgWuCoqlrfTvc9BnwSOLQ1WwvsN/C2pcC6HmKVpM2S5LXAhqq6cbB4gqY1RN1TC/3nUNIEhrkrcEmSxW1+R+CVwNfHrptqh8mPA25tb7kMOKHdHXgY8EhVrR9J9JK0aYcDr0tyN3AR3SnAj9Jd+7motRn85+8n/xi2+mcDD85kwJLmt2GOWO0NXJPkZuDLdNdYXQ58KsktwC3AHsAHWvsrgDuBNcA5wK/3HrUkDaGqTq2qpVW1DDgeuLqq3gRcA7y+NVsJXNrmL2vLtPqrq2pTN+tI0lMsmqpBVd0MvGSC8iMmaV/AydMPTZJG5t3ARUk+AHwFOLeVnwtcmGQN3ZGq42cpPknz1JSJlSQtBFV1Ld01olTVnTx5Xehgmx8Cb5jRwCQtKA5pI0mS1BMTK0mSpJ6YWEmSJPXExEqSJKknJlaSJEk9MbGSJEnqiYmVJElST0ysJEmSemJiJUmS1BMTK0mSpJ6YWEmSJPXExEqSJKknJlaSJEk9MbGSJEnqyZSJVZIdktyQ5KtJbktyeivfP8n1Se5I8tkk27Xy7dvymla/bLS7IEmSNDcMc8TqMeCIqjoYWA4cleQw4EPAmVV1IPAQcFJrfxLwUFU9HziztZMkSVrwpkysqvO9trhtmwo4AriklZ8PHNfmj23LtPojk6S3iCVJkuaooa6xSrJNkpuADcCVwDeBh6vq8dZkLbBvm98XuBeg1T8C7N5n0JIkSXPRUIlVVT1RVcuBpcChwAsnatZeJzo6VeMLkqxKsjrJ6o0bNw4bryRJ0py1WXcFVtXDwLXAYcDiJIta1VJgXZtfC+wH0OqfDTw4wbrOrqoVVbViyZIlWxa9JEnSHDLMXYFLkixu8zsCrwRuB64BXt+arQQubfOXtWVa/dVV9bQjVpIkSQvNoqmbsDdwfpJt6BKxi6vq8iRfAy5K8gHgK8C5rf25wIVJ1tAdqTp+BHFLkiTNOVMmVlV1M/CSCcrvpLveanz5D4E39BKdJEnSPOKT1yVJknpiYiVJktQTEytJkqSemFhJkiT1xMRKkiSpJyZWkiRJPTGxkiRJ6omJlSRJUk9MrCRJknpiYiVJktQTEytJC1qSHZLckOSrSW5Lcnor3z/J9UnuSPLZJNu18u3b8ppWv2w245c0v5hYSVroHgOOqKqDgeXAUUkOAz4EnFlVBwIPASe19icBD1XV84EzWztJGoqJlaQFrTrfa4vbtqmAI4BLWvn5wHFt/ti2TKs/MklmKFxJ85yJlaQFL8k2SW4CNgBXAt8EHq6qx1uTtcC+bX5f4F6AVv8IsPvMRixpvjKxkrTgVdUTVbUcWAocCrxwombtdaKjUzW+IMmqJKuTrN64cWN/wUqa16ZMrJLsl+SaJLe3Cz/f3spPS3JfkpvadMzAe05tF35+I8mrR7kDkjSsqnoYuBY4DFicZFGrWgqsa/Nrgf0AWv2zgQcnWNfZVbWiqlYsWbJk1KFLmieGOWL1OPDOqnohXWd0cpKDWt2ZVbW8TVcAtLrjgRcBRwF/nGSbEcQuSVNKsiTJ4ja/I/BK4HbgGuD1rdlK4NI2f1lbptVfXVVPO2IlSRNZNFWDqloPrG/zjya5nSevRZjIscBFVfUYcFeSNXSH3v++h3glaXPtDZzf/sF7BnBxVV2e5GvARUk+AHwFOLe1Pxe4sPVdD9L9oyhJQ5kysRrUnufyEuB64HDgbUlOAFbTHdV6iC7pum7gbYMXhUrSjKqqm+n6rfHld9L90ze+/IfAG2YgNEkL0NAXryfZGfhz4B1V9V3gY8Dz6J4Lsx748FjTCd7uhZ+SJGnBGyqxSrItXVL1qar6HEBVPdDutPkxcA5P/uf3kws/m8GLQn/CCz8lSdJCM8xdgaG75uD2qvrIQPneA81+Ebi1zV8GHN+GhdgfOBC4ob+QJUmS5qZhrrE6HHgzcEt7wB7Ae4A3JllOd5rvbuCtAFV1W5KLga/R3VF4clU90XfgkiRJc80wdwV+iYmvm7piE+85AzhjGnFJkiTNOz55XZIkqScmVpIkST0xsZIkSeqJiZUkSVJPTKwkSZJ6YmIlSZLUExMrSZKknphYSZIk9cTESpIkqScmVpIkST0xsZIkSeqJiZUkSVJPTKwkSZJ6YmIlSZLUExMrSZKknphYSZIk9WTKxCrJfkmuSXJ7ktuSvL2V75bkyiR3tNddW3mSnJVkTZKbkxwy6p2QJEmaC4Y5YvU48M6qeiFwGHBykoOAU4CrqupA4Kq2DHA0cGCbVgEf6z1qSZKkOWjKxKqq1lfVP7T5R4HbgX2BY4HzW7PzgePa/LHABdW5DlicZO/eI5ckSZpjNusaqyTLgJcA1wN7VdV66JIvYM/WbF/g3oG3rW1l49e1KsnqJKs3bty4+ZFLkiTNMUMnVkl2Bv4ceEdVfXdTTScoq6cVVJ1dVSuqasWSJUuGDUOSJGnOGiqxSrItXVL1qar6XCt+YOwUX3vd0MrXAvsNvH0psK6fcCVJkuauYe4KDHAucHtVfWSg6jJgZZtfCVw6UH5CuzvwMOCRsVOGkiRJC9miIdocDrwZuCXJTa3sPcAHgYuTnATcA7yh1V0BHAOsAX4A/EqvEUuSJM1RUyZWVfUlJr5uCuDICdoXcPI045IkSZp3fPK6JElST0ysJEmSemJiJUmS1BMTK0kLlmOdSpppJlaSFjLHOpU0o0ysJC1YjnUqaaaZWEnaKvQ51qkkTcbEStKC1/dYp22dDiQv6WlMrCQtaKMa69SB5CVNxMRK0oLlWKeSZtowYwVK0nzlWKeSZpSJlaQFy7FOJc00TwVKkiT1xMRKkiSpJyZWkiRJPZkysUryiSQbktw6UHZakvuS3NSmYwbqTm3jbH0jyatHFbgkSdJcM8wRq/OAoyYoP7OqlrfpCoA2BtfxwIvae/44yTZ9BStJkjSXTZlYVdUXgQeHXN+xwEVV9VhV3UV3y/Kh04hPkiRp3pjONVZvS3JzO1W4aysbepwth4OQJEkLzZYmVh8DngcsB9YDH27lQ4+z5XAQkiRpodmixKqqHqiqJ6rqx8A5PHm6b+hxtiRJkhaaLUqsxgYvbX4RGLtj8DLg+CTbJ9kfOBC4YXohSpIkzQ9TDmmT5DPAK4A9kqwF3g+8IslyutN8dwNvBaiq25JcDHwNeBw4uaqeGE3okiRJc8uUiVVVvXGC4nM30f4M4IzpBCVJkjQf+eR1SZKknphYSZIk9cTESpIkqScmVpIkST0xsZIkSeqJiZUkSVJPTKwkSZJ6YmIlSZLUExMrSZKknphYSZIk9cTESpIkqScmVpIkST0xsZIkSeqJiZUkSVJPTKwkSZJ6MmVileQTSTYkuXWgbLckVya5o73u2sqT5Kwka5LcnOSQUQYvSZI0lwxzxOo84KhxZacAV1XVgcBVbRngaODANq0CPtZPmJIkSXPflIlVVX0ReHBc8bHA+W3+fOC4gfILqnMdsDjJ3n0FK0mSNJdt6TVWe1XVeoD2umcr3xe4d6Dd2lb2NElWJVmdZPXGjRu3MAxJkqS5o++L1zNBWU3UsKrOrqoVVbViyZIlPYchSZI087Y0sXpg7BRfe93QytcC+w20Wwqs2/LwJEmS5o8tTawuA1a2+ZXApQPlJ7S7Aw8DHhk7ZShJs8E7myXNpGEet/AZ4O+Bf5FkbZKTgA8Cv5DkDuAX2jLAFcCdwBrgHODXRxK1JA3vPLyzWdIMWTRVg6p64yRVR07QtoCTpxuUJPWlqr6YZNm44mOBV7T584FrgXczcGczcF2SxUn29si7pGH55HVJW6Np39ksSRMxsZKkJw19Z7OPjJE0ERMrSVujad/Z7CNjJE3ExErS1sg7myWNxJQXr0vSfNbubH4FsEeStcD76e5kvrjd5XwP8IbW/ArgGLo7m38A/MqMByxpXjOxkrSgeWezpJnkqUBJkqSemFhJkiT1xMRKkiSpJyZWkiRJPTGxkiRJ6omJlSRJUk983MJ4mWhEi57UhCNjSJKkBcIjVpIkST0xsZIkSerJtE4FJrkbeBR4Ani8qlYk2Q34LLAMuBv4pap6aHphSpIkzX19XGP181X17YHlU4CrquqDSU5py+/uYTuSpE3I6aO7RrTe7zWi0jBGcSrwWOD8Nn8+cNwItiFJkjTnTDexKuALSW5MsqqV7VVV6wHa657T3IYkSdK8MN1TgYdX1bokewJXJvn6sG9sidgqgOc+97nTDEOSJGn2TeuIVVWta68bgM8DhwIPJNkboL1umOS9Z1fViqpasWTJkumEIUmSNCdscWKVZKcku4zNA68CbgUuA1a2ZiuBS6cbpCRJ0nwwnVOBewGfT/ek8kXAp6vqL5J8Gbg4yUnAPcAbph+mJEnS3LfFiVVV3QkcPEH5d4AjpxOUJEnSfOST1yVJknriIMwzyQGeJUla0DxiJUmS1BMTK0mSpJ6YWEmSJPXEa6w0Na8NkyRpKCZWC8Uokx9JkjQUTwVKkiT1xMRKkiSpJyZWkiRJPTGxkiRJ6omJlSRJUk9MrCRJknpiYiVJktQTn2MlSZpSTh/ds/Lq/T4oWAvHyI5YJTkqyTeSrElyyqi2I0l9s/+StKVGklgl2Qb4I+Bo4CDgjUkOGsW2JKlP9l+SpmNUR6wOBdZU1Z1V9c/ARcCxI9qW5rNkdJO0Zey/JG2xUV1jtS9w78DyWuClI9qWtLDM16Rw4Qyobf81w0Z5/dYojfLaMK9pm9h8+FxGlVhNtOdPiTjJKmBVW/xekm8Mue49gG9PI7a5yv3q2+gTFL+zQZv/ef/UZm9jZkzZf8EW9WFz4edltmOY7e33GkNO26I+ZtY/g5yW2Y5htrc/YQyb+X1O2n+NKrFaC+w3sLwUWDfYoKrOBs7e3BUnWV1VK6YX3tzjfs0/C3XfFup+bYYp+y/Y/D5sLnyusx3DbG9/LsQw29ufCzHM9vZHHcOorrH6MnBgkv2TbAccD1w2om1JUp/svyRtsZEcsaqqx5O8DfhLYBvgE1V12yi2JUl9sv+SNB0je0BoVV0BXDGCVW/26cN5wv2afxbqvi3U/RraiPqvufC5znYMs719mP0YZnv7MMAQHvUAAAXMSURBVPsxzPb2YYQxpBbOnTySJEmzyrECJUmSejJvEquFNsREkruT3JLkpiSrW9luSa5Mckd73XW245xKkk8k2ZDk1oGyCfcjnbPad3hzkkNmL/JNm2S/TktyX/vObkpyzEDdqW2/vpHk1bMT9dSS7JfkmiS3J7ktydtb+bz/zuayme6/Nvd7HnEs2yT5SpLL2/L+Sa5vMXy23SAwqm0vTnJJkq+3z+Jfz/RnkOQ32ndwa5LPJNlh1J/BbPfLk2z/f7Tv4eYkn0+yeKCu1/5zou0P1P1WkkqyR1vuv4+rqjk/0V1A+k3gAGA74KvAQbMd1zT36W5gj3Flvwec0uZPAT4023EOsR8/BxwC3DrVfgDHAP+X7jlBhwHXz3b8m7lfpwG/NUHbg9rP5PbA/u1ndZvZ3odJ9mtv4JA2vwvwjy3+ef+dzdVpNvqvzf2eRxzLbwKfBi5vyxcDx7f5PwH+4wi3fT7wlja/HbB4Jj8DuofN3gXsOLDvJ476M5jtfnmS7b8KWNTmPzSw/d77z4m238r3o7sp5Vu0v7+j2P/5csRqaxli4li6joD2etwsxjKUqvoi8OC44sn241jggupcByxOsvfMRLp5JtmvyRwLXFRVj1XVXcAaup/ZOaeq1lfVP7T5R4Hb6Tr/ef+dzWEz3n9twfc8EkmWAq8BPt6WAxwBXDLqGJI8i+4P7LkAVfXPVfUwM9/PLgJ2TLIIeCawnhF/BrPdL0+0/ar6QlU93havo3s+3Nj2e+0/N9F/nwm8i6c+8Lf3/Z8vidVEQ0zsO0ux9KWALyS5Md0TnAH2qqr10HWMwJ6zFt30TLYfC+F7fFs7XPyJgVMI83K/kiwDXgJcz8L+zmbbrH6GQ37Po/JRuj9kP27LuwMPD/yBHeVncQCwEfhkOxX58SQ7MYOfQVXdB/w+cA9dQvUIcCMz9xkMmku/4/+B7ijRjG0/yeuA+6rqq+Oqet/+fEmshhpiYp45vKoOAY4GTk7yc7Md0AyY79/jx4DnAcvpOskPt/J5t19Jdgb+HHhHVX13U00nKJvT+zYHzdpnuBnf8yi2/VpgQ1XdOFg8QdNRfRaL6E4HfayqXgJ8n+4U2Ixp/3wdS3eKax9gJ7o+f7zZ/J2a0Z/PJO8FHgc+NVPbT/JM4L3A+yaq7nv78yWxGmqIifmkqta11w3A5+kOfT4wdgiyvW6YvQinZbL9mNffY1U9UFVPVNWPgXN48nD1vNqvJNvS/bH9VFV9rhUvyO9sjpiVz3Azv+dROBx4XZK76U5/HkF3BGtxOy0Go/0s1gJrq+r6tnwJXaI1k5/BK4G7qmpjVf0I+BzwMmbuMxg067/jSVYCrwXeVO0Cpxna/vPoktuvtp/HpcA/JHnOKLY/XxKrBTXERJKdkuwyNk93Ud+tdPu0sjVbCVw6OxFO22T7cRlwQrsL4zDgkbFD0/PBuPPuv0j3nUG3X8cn2T7J/sCBwA0zHd8w2jUu5wK3V9VHBqoW5Hc2R8x4/7UF33PvqurUqlpaVcvo9vnqqnoTcA3w+lHHUFX3A/cm+Ret6Ejga8xsP3sPcFiSZ7bvZCyGGfkMxpnV3/EkRwHvBl5XVT8YF9dI+8+quqWq9qyqZe3ncS3dzR33M4r9n+7V7zM10V25/490dwy8d7bjmea+HEB3F8RXgdvG9ofu+oOrgDva626zHesQ+/IZutNiP2o/rCdNth90h1z/qH2HtwArZjv+zdyvC1vcN7dfxr0H2r+37dc3gKNnO/5N7NfL6Q5z3wzc1KZjFsJ3Npenme6/Nvd7noF4XsGTdwUeQPeHcw3wZ8D2I9zucmB1+xz+F7DrTH8GwOnA1+n+EbuQ7u63kX4Gs90vT7L9NXTXMo39PP7JQPte+8+Jtj+u/m6evCuw9/33yeuSJEk9mS+nAiVJkuY8EytJkqSemFhJkiT1xMRKkiSpJyZWkiRJPTGxkiRJ6omJlSRJUk9MrCRJknry/wADGgLfr7mTbwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Number of words in a comment\n",
    "df_train['word_count'] = df_train['CONTENT'].apply(lambda x: len(str(x).split()))\n",
    "print(df_train[df_train['CLASS']==1]['word_count'].mean()) #Ad tweets\n",
    "print(df_train[df_train['CLASS']==0]['word_count'].mean()) #Non-Ad tweets\n",
    "\n",
    "#Plotting word count\n",
    "fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,4))\n",
    "train_words=df_train[df_train['CLASS']==1]['word_count']\n",
    "ax1.hist(train_words,color='red')\n",
    "ax1.set_title('Ad comments')\n",
    "train_words=df_train[df_train['CLASS']==0]['word_count']\n",
    "ax2.hist(train_words,color='green')\n",
    "ax2.set_title('Non-Ad comments')\n",
    "fig.suptitle('Words per comment')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "138.26621160409556\n",
      "49.08406304728546\n"
     ]
    }
   ],
   "source": [
    "#Number of characters in a comment\n",
    "df_train['char_count'] = df_train['CONTENT'].apply(lambda x: len(str(x)))\n",
    "print(df_train[df_train['CLASS']==1]['char_count'].mean()) #Ad comments\n",
    "print(df_train[df_train['CLASS']==0]['char_count'].mean()) #Non-Ad comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COMMENT_ID</th>\n",
       "      <th>AUTHOR</th>\n",
       "      <th>DATE</th>\n",
       "      <th>CONTENT</th>\n",
       "      <th>CLASS</th>\n",
       "      <th>word_count</th>\n",
       "      <th>char_count</th>\n",
       "      <th>CLEAN_CONTENT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LZQPQhLyRh80UYxNuaDWhIGQYNQ96IuCg-AYWqNPjpU</td>\n",
       "      <td>Julius NM</td>\n",
       "      <td>2013-11-07T06:20:48</td>\n",
       "      <td>Huh, anyway check out this you[tube] channel: ...</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>56</td>\n",
       "      <td>huh anyway check tube channel kobyoshi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>z13jhp0bxqncu512g22wvzkasxmvvzjaz04</td>\n",
       "      <td>ElNino Melendez</td>\n",
       "      <td>2013-11-09T08:28:43</td>\n",
       "      <td>me shaking my sexy ass on my channel enjoy ^_^ ﻿</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>48</td>\n",
       "      <td>shake sexy as channel enjoy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>z13fwbwp1oujthgqj04chlngpvzmtt3r3dw</td>\n",
       "      <td>GsMega</td>\n",
       "      <td>2013-11-10T16:05:38</td>\n",
       "      <td>watch?v=vtaRGgvGtWQ   Check this out .﻿</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>39</td>\n",
       "      <td>watch v vtarggvgtwq check</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>z13lfzdo5vmdi1cm123te5uz2mqig1brz04</td>\n",
       "      <td>ferleck ferles</td>\n",
       "      <td>2013-11-27T21:39:24</td>\n",
       "      <td>Subscribe to my channel ﻿</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>25</td>\n",
       "      <td>subscribe channel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>z12avveb4xqiirsix04chxviiljryduwxg0</td>\n",
       "      <td>BeBe Burkey</td>\n",
       "      <td>2013-11-28T16:30:13</td>\n",
       "      <td>and u should.d check my channel and tell me wh...</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>67</td>\n",
       "      <td>u check channel tell next</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    COMMENT_ID           AUTHOR  \\\n",
       "0  LZQPQhLyRh80UYxNuaDWhIGQYNQ96IuCg-AYWqNPjpU        Julius NM   \n",
       "1          z13jhp0bxqncu512g22wvzkasxmvvzjaz04  ElNino Melendez   \n",
       "2          z13fwbwp1oujthgqj04chlngpvzmtt3r3dw           GsMega   \n",
       "3          z13lfzdo5vmdi1cm123te5uz2mqig1brz04   ferleck ferles   \n",
       "4          z12avveb4xqiirsix04chxviiljryduwxg0      BeBe Burkey   \n",
       "\n",
       "                  DATE                                            CONTENT  \\\n",
       "0  2013-11-07T06:20:48  Huh, anyway check out this you[tube] channel: ...   \n",
       "1  2013-11-09T08:28:43   me shaking my sexy ass on my channel enjoy ^_^ ﻿   \n",
       "2  2013-11-10T16:05:38            watch?v=vtaRGgvGtWQ   Check this out .﻿   \n",
       "3  2013-11-27T21:39:24                          Subscribe to my channel ﻿   \n",
       "4  2013-11-28T16:30:13  and u should.d check my channel and tell me wh...   \n",
       "\n",
       "   CLASS  word_count  char_count                           CLEAN_CONTENT  \n",
       "0      1           8          56  huh anyway check tube channel kobyoshi  \n",
       "1      1          11          48             shake sexy as channel enjoy  \n",
       "2      1           5          39               watch v vtarggvgtwq check  \n",
       "3      1           5          25                       subscribe channel  \n",
       "4      1          14          67               u check channel tell next  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Text pre-processing(stop-word,stemming,lemmatization)\n",
    "#convert to lowercase, strip and remove punctuations\n",
    "def preprocess(CONTENT):\n",
    "    CONTENT=CONTENT.lower() \n",
    "    CONTENT=CONTENT.strip()  \n",
    "    CONTENT=re.compile('<.*?>').sub('', CONTENT) \n",
    "    CONTENT = re.compile('[%s]' % re.escape(string.punctuation)).sub(' ', CONTENT)  \n",
    "    CONTENT = re.sub('\\s+', ' ', CONTENT)  \n",
    "    CONTENT = re.sub(r'\\[[0-9]*\\]',' ',CONTENT) \n",
    "    CONTENT=re.sub(r'[^\\w\\s]', '', str(CONTENT).lower().strip())\n",
    "    CONTENT = re.sub(r'\\d',' ',CONTENT) \n",
    "    CONTENT = re.sub(r'\\s+',' ',CONTENT) \n",
    "    return CONTENT\n",
    "\n",
    " \n",
    "# STOPWORD REMOVAL\n",
    "def stopword(string):\n",
    "    a= [i for i in string.split() if i not in stopwords.words('english')]\n",
    "    return ' '.join(a)\n",
    "#LEMMATIZATION\n",
    "# Initialize the lemmatizer\n",
    "wl = WordNetLemmatizer()\n",
    " \n",
    "# This is a helper function to map NTLK position tags\n",
    "def get_wordnet_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN\n",
    "# Tokenize the sentence\n",
    "def lemmatizer(string):\n",
    "    word_pos_tags = nltk.pos_tag(word_tokenize(string)) # Get position tags\n",
    "    a=[wl.lemmatize(tag[0], get_wordnet_pos(tag[1])) for idx, tag in enumerate(word_pos_tags)] # Map the position tag and lemmatize the word/token\n",
    "    return \" \".join(a)\n",
    "\n",
    "#Final\n",
    "def finalpreprocess(string):\n",
    "    return lemmatizer(stopword(preprocess(string)))\n",
    "df_train['CLEAN_CONTENT'] = df_train['CONTENT'].apply(lambda x: finalpreprocess(x))\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Word2vec model\n",
    "df_train['CLEAN_CONTENT_tok']=[nltk.word_tokenize(i) for i in df_train['CLEAN_CONTENT']] #convert preprocessed sentence to tokenized sentence\n",
    "model = Word2Vec(df_train['CLEAN_CONTENT_tok'],min_count=1) \n",
    "\n",
    "w2v = dict(zip(model.wv.index_to_key, model.wv))  #combination of word and its vector\n",
    "#for converting sentence to vectors/numbers from word vectors result by Word2Vec\n",
    "class MeanEmbeddingVectorizer(object):\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    "        # if a text is empty we should return a vector of zeros\n",
    "        # with the same dimensionality as all the other vectors\n",
    "        self.dim = len(next(iter(word2vec.values())))\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return np.array([\n",
    "            np.mean([self.word2vec[w] for w in words if w in self.word2vec]\n",
    "                    or [np.zeros(self.dim)], axis=0)\n",
    "            for words in X\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#VECTORIZATION\n",
    "#Splitting training data into train and test\n",
    "X_train, X_test, y_train, y_test = train_test_split(df_train[\"CLEAN_CONTENT\"],df_train[\"CLASS\"],test_size=0.25,shuffle=True)\n",
    "#Word2Vec\n",
    "# Word2Vec runs on tokenized sentences\n",
    "X_train_tok= [nltk.word_tokenize(i) for i in X_train]  \n",
    "X_test_tok= [nltk.word_tokenize(i) for i in X_test]\n",
    "\n",
    "#tfidf runs on non-tokenized sentences unlike word2vec\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True)\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train) \n",
    "X_test_vectors_tfidf = tfidf_vectorizer.transform(X_test)\n",
    "\n",
    "#Word2vec\n",
    "# Fit and transform\n",
    "modelw = MeanEmbeddingVectorizer(w2v)\n",
    "X_train_vectors_w2v = modelw.transform(X_train_tok)\n",
    "X_test_vectors_w2v = modelw.transform(X_test_tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.97      0.95       137\n",
      "           1       0.97      0.93      0.95       153\n",
      "\n",
      "    accuracy                           0.95       290\n",
      "   macro avg       0.95      0.95      0.95       290\n",
      "weighted avg       0.95      0.95      0.95       290\n",
      "\n",
      "Confusion Matrix: [[133   4]\n",
      " [ 11 142]]\n",
      "AUC: 0.9824912933543247\n"
     ]
    }
   ],
   "source": [
    "#MACHINE LEARNING\n",
    "#LR on tfidf\n",
    "#FITTING THE CLASSIFICATION MODEL using Logistic Regression(tf-idf)\n",
    "lr_tfidf=LogisticRegression(solver = 'liblinear', C=10, penalty = 'l2')\n",
    "lr_tfidf.fit(X_train_vectors_tfidf, y_train)  \n",
    "#Predict y value for test dataset\n",
    "y_predict = lr_tfidf.predict(X_test_vectors_tfidf)\n",
    "y_prob = lr_tfidf.predict_proba(X_test_vectors_tfidf)[:,1]\n",
    "print(classification_report(y_test,y_predict))\n",
    "print('Confusion Matrix:',confusion_matrix(y_test, y_predict))\n",
    " \n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('AUC:', roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.65      0.69      0.67       137\n",
      "           1       0.71      0.67      0.69       153\n",
      "\n",
      "    accuracy                           0.68       290\n",
      "   macro avg       0.68      0.68      0.68       290\n",
      "weighted avg       0.68      0.68      0.68       290\n",
      "\n",
      "Confusion Matrix: [[ 94  43]\n",
      " [ 50 103]]\n",
      "AUC: 0.7514908639854969\n"
     ]
    }
   ],
   "source": [
    "#LR on w2v\n",
    "#FITTING THE CLASSIFICATION MODEL using Logistic Regression (W2v)\n",
    "lr_w2v=LogisticRegression(solver = 'liblinear', C=10, penalty = 'l2')\n",
    "lr_w2v.fit(X_train_vectors_w2v, y_train)  #model\n",
    "#Predict y value for test dataset\n",
    "y_predict = lr_w2v.predict(X_test_vectors_w2v)\n",
    "y_prob = lr_w2v.predict_proba(X_test_vectors_w2v)[:,1]\n",
    "print(classification_report(y_test,y_predict))\n",
    "print('Confusion Matrix:',confusion_matrix(y_test, y_predict))\n",
    " \n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('AUC:', roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.89      0.90       137\n",
      "           1       0.90      0.93      0.92       153\n",
      "\n",
      "    accuracy                           0.91       290\n",
      "   macro avg       0.91      0.91      0.91       290\n",
      "weighted avg       0.91      0.91      0.91       290\n",
      "\n",
      "Confusion Matrix: [[122  15]\n",
      " [ 11 142]]\n",
      "AUC: 0.9772434521253757\n"
     ]
    }
   ],
   "source": [
    "#Naive Bayes\n",
    "#tfidf\n",
    "#FITTING THE CLASSIFICATION MODEL using Naive Bayes(tf-idf)\n",
    "nb_tfidf = MultinomialNB()\n",
    "nb_tfidf.fit(X_train_vectors_tfidf, y_train)  \n",
    "#Predict y value for test dataset\n",
    "y_predict = nb_tfidf.predict(X_test_vectors_tfidf)\n",
    "y_prob = nb_tfidf.predict_proba(X_test_vectors_tfidf)[:,1]\n",
    "print(classification_report(y_test,y_predict))\n",
    "print('Confusion Matrix:',confusion_matrix(y_test, y_predict))\n",
    " \n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('AUC:', roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   ID                                   COMMENT_ID            AUTHOR  \\\n",
      "0   0  LZQPQhLyRh_C2cTtd9MvFRJedxydaVW-2sNg5Diuo4A       adam riyati   \n",
      "1   1  LZQPQhLyRh9MSZYnf8djyk0gEF9BHDPYrrK-qCczIY8  Evgeny Murashkin   \n",
      "2   2  LZQPQhLyRh9-wNRtlZDM90f1k0BrdVdJyN_YsaSwfxc      Jason Haddad   \n",
      "3   3        z122wfnzgt30fhubn04cdn3xfx2mxzngsl40k      Bob Kanowski   \n",
      "4   4          z13ttt1jcraqexk2o234ghbgzxymz1zzi04              Cony   \n",
      "\n",
      "                  DATE                                            CONTENT  \\\n",
      "0  2013-11-07T12:37:15  Hey guys check out my new channel and our firs...   \n",
      "1  2013-11-08T17:34:21             just for test I have to say murdev.com   \n",
      "2  2013-11-26T02:55:11  Hey, check out my new website!! This site is a...   \n",
      "3  2013-11-28T12:33:27  i turned it on mute as soon is i came on i jus...   \n",
      "4  2013-11-28T16:01:47    You should check my channel for Funny VIDEOS!!﻿   \n",
      "\n",
      "                                       CLEAN_CONTENT  predict_prob  target  \\\n",
      "0  hey guy check new channel first vid u monkey m...      0.999726       1   \n",
      "1                                test say murdev com      0.883215       1   \n",
      "2  hey check new website site kid stuff kidsmedia...      0.996906       1   \n",
      "3                turn mute soon come want check view      0.081602       0   \n",
      "4                          check channel funny video      0.968579       1   \n",
      "\n",
      "   CLASS  \n",
      "0      1  \n",
      "1      1  \n",
      "2      1  \n",
      "3      0  \n",
      "4      1  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>CLASS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>794</th>\n",
       "      <td>794</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>795</th>\n",
       "      <td>795</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796</th>\n",
       "      <td>796</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>797</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>798</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>799 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ID  CLASS\n",
       "0      0      1\n",
       "1      1      1\n",
       "2      2      1\n",
       "3      3      0\n",
       "4      4      1\n",
       "..   ...    ...\n",
       "794  794      1\n",
       "795  795      0\n",
       "796  796      0\n",
       "797  797      0\n",
       "798  798      0\n",
       "\n",
       "[799 rows x 2 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Applying LR on test set(since it is the best fit)\n",
    "df_test['CLEAN_CONTENT'] = df_test['CONTENT'].apply(lambda x: finalpreprocess(x)) #preprocess the data\n",
    "X_test=df_test['CLEAN_CONTENT'] \n",
    "X_vector=tfidf_vectorizer.transform(X_test) #converting X_test to vector\n",
    "y_predict = lr_tfidf.predict(X_vector)      #use the trained model on X_vector\n",
    "y_prob = lr_tfidf.predict_proba(X_vector)[:,1]\n",
    "df_test['predict_prob']= y_prob\n",
    "df_test['CLASS']= y_predict\n",
    "print(df_test.head())\n",
    "final=df_test[['ID',\"CLASS\"]].reset_index(drop=True)\n",
    "\n",
    "final.to_csv('submission1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}