Twitter Sentiment Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import required libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn import svm\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import preprocessor as p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tweet-preprocessor in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (0.6.0)\r\n"
     ]
    }
   ],
   "source": [
    "#!pip install tweet-preprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import data\n",
    "train = pd.read_csv('train_E6oV3lV.csv')\n",
    "test = pd.read_csv('test_tweets_anuFYb8.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>@user when a father is dysfunctional and is s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>@user @user thanks for #lyft credit i can't us...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>bihday your majesty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>#model   i love u take with u all the time in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>factsguide: society now    #motivation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31957</th>\n",
       "      <td>31958</td>\n",
       "      <td>0</td>\n",
       "      <td>ate @user isz that youuu?ðððððð...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31958</th>\n",
       "      <td>31959</td>\n",
       "      <td>0</td>\n",
       "      <td>to see nina turner on the airwaves trying to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31959</th>\n",
       "      <td>31960</td>\n",
       "      <td>0</td>\n",
       "      <td>listening to sad songs on a monday morning otw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31960</th>\n",
       "      <td>31961</td>\n",
       "      <td>1</td>\n",
       "      <td>@user #sikh #temple vandalised in in #calgary,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31961</th>\n",
       "      <td>31962</td>\n",
       "      <td>0</td>\n",
       "      <td>thank you @user for you follow</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31962 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  label                                              tweet\n",
       "0          1      0   @user when a father is dysfunctional and is s...\n",
       "1          2      0  @user @user thanks for #lyft credit i can't us...\n",
       "2          3      0                                bihday your majesty\n",
       "3          4      0  #model   i love u take with u all the time in ...\n",
       "4          5      0             factsguide: society now    #motivation\n",
       "...      ...    ...                                                ...\n",
       "31957  31958      0  ate @user isz that youuu?ðððððð...\n",
       "31958  31959      0    to see nina turner on the airwaves trying to...\n",
       "31959  31960      0  listening to sad songs on a monday morning otw...\n",
       "31960  31961      1  @user #sikh #temple vandalised in in #calgary,...\n",
       "31961  31962      0                   thank you @user for you follow  \n",
       "\n",
       "[31962 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#eda\n",
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31963</td>\n",
       "      <td>#studiolife #aislife #requires #passion #dedic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>31964</td>\n",
       "      <td>@user #white #supremacists want everyone to s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31965</td>\n",
       "      <td>safe ways to heal your #acne!!    #altwaystohe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31966</td>\n",
       "      <td>is the hp and the cursed child book up for res...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31967</td>\n",
       "      <td>3rd #bihday to my amazing, hilarious #nephew...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17192</th>\n",
       "      <td>49155</td>\n",
       "      <td>thought factory: left-right polarisation! #tru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17193</th>\n",
       "      <td>49156</td>\n",
       "      <td>feeling like a mermaid ð #hairflip #neverre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17194</th>\n",
       "      <td>49157</td>\n",
       "      <td>#hillary #campaigned today in #ohio((omg)) &amp;am...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17195</th>\n",
       "      <td>49158</td>\n",
       "      <td>happy, at work conference: right mindset leads...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17196</th>\n",
       "      <td>49159</td>\n",
       "      <td>my   song \"so glad\" free download!  #shoegaze ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17197 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                              tweet\n",
       "0      31963  #studiolife #aislife #requires #passion #dedic...\n",
       "1      31964   @user #white #supremacists want everyone to s...\n",
       "2      31965  safe ways to heal your #acne!!    #altwaystohe...\n",
       "3      31966  is the hp and the cursed child book up for res...\n",
       "4      31967    3rd #bihday to my amazing, hilarious #nephew...\n",
       "...      ...                                                ...\n",
       "17192  49155  thought factory: left-right polarisation! #tru...\n",
       "17193  49156  feeling like a mermaid ð #hairflip #neverre...\n",
       "17194  49157  #hillary #campaigned today in #ohio((omg)) &am...\n",
       "17195  49158  happy, at work conference: right mindset leads...\n",
       "17196  49159  my   song \"so glad\" free download!  #shoegaze ...\n",
       "\n",
       "[17197 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of tweets that are not racist/sexist :  29720\n",
      "The number of tweets that are racist/sexist :  2242\n"
     ]
    }
   ],
   "source": [
    "#racist/sexist tweets in train\n",
    "print('The number of tweets that are not racist/sexist : ', sum(train.label == 0))\n",
    "print('The number of tweets that are racist/sexist : ', sum(train.label == 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id       0\n",
       "label    0\n",
       "tweet    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#null values?\n",
    "train.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data cleaning\n",
    "#set up special charecters/punctuations we want to be replaced using regular expression\n",
    "re_without_space = re.compile(\"(\\.)|(\\,)|(\\{)|(\\})|(\\()|(\\))|(\\[)|(\\])|(\\;)|(\\:)|(\\!)|(\\`)|(\\')|(\\\")|(\\%)|(\\$)|(\\<)|(\\>)|(\\?)|(\\|)|\")\n",
    "re_with_space = re.compile(\"(<br\\s/><br\\s/?)|(-)|(/)|(:).\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#defining a function to clean the dataset using tweet preprocessor and re\n",
    "def cleantweet(tweet):\n",
    "        temp = p.clean(tweet)\n",
    "        temp = re_without_space.sub('', temp.lower())\n",
    "        temp = re_with_space.sub(' ', temp)\n",
    "        temp = re.sub('[0-9]', '', temp)\n",
    "        temp = re.sub('_', '', temp)\n",
    "        return temp\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cleaning the train and test datasets\n",
    "train['clean'] = train['tweet'].apply(cleantweet)\n",
    "test['clean'] = test['tweet'].apply(cleantweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet</th>\n",
       "      <th>clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>@user when a father is dysfunctional and is s...</td>\n",
       "      <td>when a father is dysfunctional and is so selfi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>@user @user thanks for #lyft credit i can't us...</td>\n",
       "      <td>thanks for credit i cant use cause they dont o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>bihday your majesty</td>\n",
       "      <td>bihday your majesty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>#model   i love u take with u all the time in ...</td>\n",
       "      <td>i love u take with u all the time in ur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>factsguide: society now    #motivation</td>\n",
       "      <td>factsguide society now</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31957</th>\n",
       "      <td>31958</td>\n",
       "      <td>0</td>\n",
       "      <td>ate @user isz that youuu?ðððððð...</td>\n",
       "      <td>ate isz that youuu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31958</th>\n",
       "      <td>31959</td>\n",
       "      <td>0</td>\n",
       "      <td>to see nina turner on the airwaves trying to...</td>\n",
       "      <td>to see nina turner on the airwaves trying to w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31959</th>\n",
       "      <td>31960</td>\n",
       "      <td>0</td>\n",
       "      <td>listening to sad songs on a monday morning otw...</td>\n",
       "      <td>listening to sad songs on a monday morning otw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31960</th>\n",
       "      <td>31961</td>\n",
       "      <td>1</td>\n",
       "      <td>@user #sikh #temple vandalised in in #calgary,...</td>\n",
       "      <td>vandalised in in  condemns act</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31961</th>\n",
       "      <td>31962</td>\n",
       "      <td>0</td>\n",
       "      <td>thank you @user for you follow</td>\n",
       "      <td>thank you for you follow</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31962 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  label                                              tweet  \\\n",
       "0          1      0   @user when a father is dysfunctional and is s...   \n",
       "1          2      0  @user @user thanks for #lyft credit i can't us...   \n",
       "2          3      0                                bihday your majesty   \n",
       "3          4      0  #model   i love u take with u all the time in ...   \n",
       "4          5      0             factsguide: society now    #motivation   \n",
       "...      ...    ...                                                ...   \n",
       "31957  31958      0  ate @user isz that youuu?ðððððð...   \n",
       "31958  31959      0    to see nina turner on the airwaves trying to...   \n",
       "31959  31960      0  listening to sad songs on a monday morning otw...   \n",
       "31960  31961      1  @user #sikh #temple vandalised in in #calgary,...   \n",
       "31961  31962      0                   thank you @user for you follow     \n",
       "\n",
       "                                                   clean  \n",
       "0      when a father is dysfunctional and is so selfi...  \n",
       "1      thanks for credit i cant use cause they dont o...  \n",
       "2                                    bihday your majesty  \n",
       "3                i love u take with u all the time in ur  \n",
       "4                                 factsguide society now  \n",
       "...                                                  ...  \n",
       "31957                                 ate isz that youuu  \n",
       "31958  to see nina turner on the airwaves trying to w...  \n",
       "31959  listening to sad songs on a monday morning otw...  \n",
       "31960                     vandalised in in  condemns act  \n",
       "31961                           thank you for you follow  \n",
       "\n",
       "[31962 rows x 4 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "label=train.pop('label')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "#nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemma = WordNetLemmatizer()\n",
    "\n",
    "\n",
    "def lemmatizing(text):\n",
    "    text_token = word_tokenize(text)\n",
    "    stemmed_words = [lemma.lemmatize(word) for word in text_token]\n",
    "    clean_text = \" \".join(stemmed_words)\n",
    "    clean_text = clean_text.replace('   ', ' ')\n",
    "    clean_text = clean_text.replace('  ', ' ')\n",
    "    return clean_text   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['clean2'] = train['clean'].apply(lemmatizing)\n",
    "test['clean2'] = test['clean'].apply(lemmatizing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet</th>\n",
       "      <th>clean</th>\n",
       "      <th>clean2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31963</td>\n",
       "      <td>#studiolife #aislife #requires #passion #dedic...</td>\n",
       "      <td>to find</td>\n",
       "      <td>to find</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>31964</td>\n",
       "      <td>@user #white #supremacists want everyone to s...</td>\n",
       "      <td>want everyone to see the new and heres why</td>\n",
       "      <td>want everyone to see the new and here why</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31965</td>\n",
       "      <td>safe ways to heal your #acne!!    #altwaystohe...</td>\n",
       "      <td>safe ways to heal your</td>\n",
       "      <td>safe way to heal your</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31966</td>\n",
       "      <td>is the hp and the cursed child book up for res...</td>\n",
       "      <td>is the hp and the cursed child book up for res...</td>\n",
       "      <td>is the hp and the cursed child book up for res...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31967</td>\n",
       "      <td>3rd #bihday to my amazing, hilarious #nephew...</td>\n",
       "      <td>rd to my amazing hilarious eli ahmir uncle dav...</td>\n",
       "      <td>rd to my amazing hilarious eli ahmir uncle dav...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17192</th>\n",
       "      <td>49155</td>\n",
       "      <td>thought factory: left-right polarisation! #tru...</td>\n",
       "      <td>thought factory left right polarisation &amp;gt</td>\n",
       "      <td>thought factory left right polarisation &amp; gt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17193</th>\n",
       "      <td>49156</td>\n",
       "      <td>feeling like a mermaid ð #hairflip #neverre...</td>\n",
       "      <td>feeling like a mermaid</td>\n",
       "      <td>feeling like a mermaid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17194</th>\n",
       "      <td>49157</td>\n",
       "      <td>#hillary #campaigned today in #ohio((omg)) &amp;am...</td>\n",
       "      <td>today in omg &amp;amp used words like assets&amp;ampli...</td>\n",
       "      <td>today in omg &amp; amp used word like asset &amp; ampl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17195</th>\n",
       "      <td>49158</td>\n",
       "      <td>happy, at work conference: right mindset leads...</td>\n",
       "      <td>happy at work conference right mindset leads t...</td>\n",
       "      <td>happy at work conference right mindset lead to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17196</th>\n",
       "      <td>49159</td>\n",
       "      <td>my   song \"so glad\" free download!  #shoegaze ...</td>\n",
       "      <td>my song so glad free download</td>\n",
       "      <td>my song so glad free download</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17197 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                              tweet  \\\n",
       "0      31963  #studiolife #aislife #requires #passion #dedic...   \n",
       "1      31964   @user #white #supremacists want everyone to s...   \n",
       "2      31965  safe ways to heal your #acne!!    #altwaystohe...   \n",
       "3      31966  is the hp and the cursed child book up for res...   \n",
       "4      31967    3rd #bihday to my amazing, hilarious #nephew...   \n",
       "...      ...                                                ...   \n",
       "17192  49155  thought factory: left-right polarisation! #tru...   \n",
       "17193  49156  feeling like a mermaid ð #hairflip #neverre...   \n",
       "17194  49157  #hillary #campaigned today in #ohio((omg)) &am...   \n",
       "17195  49158  happy, at work conference: right mindset leads...   \n",
       "17196  49159  my   song \"so glad\" free download!  #shoegaze ...   \n",
       "\n",
       "                                                   clean  \\\n",
       "0                                                to find   \n",
       "1             want everyone to see the new and heres why   \n",
       "2                               safe ways to heal your     \n",
       "3      is the hp and the cursed child book up for res...   \n",
       "4      rd to my amazing hilarious eli ahmir uncle dav...   \n",
       "...                                                  ...   \n",
       "17192        thought factory left right polarisation &gt   \n",
       "17193                             feeling like a mermaid   \n",
       "17194  today in omg &amp used words like assets&ampli...   \n",
       "17195  happy at work conference right mindset leads t...   \n",
       "17196                      my song so glad free download   \n",
       "\n",
       "                                                  clean2  \n",
       "0                                                to find  \n",
       "1              want everyone to see the new and here why  \n",
       "2                                  safe way to heal your  \n",
       "3      is the hp and the cursed child book up for res...  \n",
       "4      rd to my amazing hilarious eli ahmir uncle dav...  \n",
       "...                                                  ...  \n",
       "17192       thought factory left right polarisation & gt  \n",
       "17193                             feeling like a mermaid  \n",
       "17194  today in omg & amp used word like asset & ampl...  \n",
       "17195  happy at work conference right mindset lead to...  \n",
       "17196                      my song so glad free download  \n",
       "\n",
       "[17197 rows x 4 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "3    0\n",
       "4    0\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "id=test.pop('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17197,)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer(ngram_range=(1,3), min_df=5, max_df=0.8,stop_words = 'english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(max_df=0.8, min_df=5, ngram_range=(1, 3), stop_words='english')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_tfidf = tfidf.fit_transform(x)\n",
    "#X_test_tfidf = tfidf.transform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<31962x6037 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 160063 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#spliting the train data for train and test split cross validation\n",
    "x_train1, x_test1, y_train1, y_test1 = train_test_split(X_train_tfidf,y,stratify=y,test_size=0.3,random_state=2,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "\n",
    "svm2 = svm.SVC(kernel='linear', probability = True)\n",
    "model_2= svm2.fit(x_train1,y_train1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tfidf=model_2.predict(x_test1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.492972972972973"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "\n",
    "f1_score(y_test1, pred_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[8892,   24],\n",
       "       [ 445,  228]])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test1, pred_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_tfidf = tfidf.transform(test['clean2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tfidf_act=model_2.predict(X_test_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_tfidf_act"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_submission = pd.DataFrame({'id':id,'label': pred_tfidf_act})\n",
    "my_submission.to_csv('twitter_5.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gensim\n",
      "  Downloading gensim-4.0.1-cp38-cp38-macosx_10_9_x86_64.whl (23.9 MB)\n",
      "\u001b[K     |████████████████████████████████| 23.9 MB 331 kB/s eta 0:00:01     |██████████████▋                 | 10.9 MB 502 kB/s eta 0:00:26     |██████████████████████████████▊ | 23.0 MB 194 kB/s eta 0:00:05\n",
      "\u001b[?25hCollecting smart-open>=1.8.1\n",
      "  Downloading smart_open-5.1.0-py3-none-any.whl (57 kB)\n",
      "\u001b[K     |████████████████████████████████| 57 kB 536 kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.11.3 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from gensim) (1.20.2)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from gensim) (1.6.2)\n",
      "Installing collected packages: smart-open, gensim\n",
      "Successfully installed gensim-4.0.1 smart-open-5.1.0\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages/gensim/similarities/__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package <https://pypi.org/project/python-Levenshtein/> is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_tweet = train['clean2'].apply(lambda x: x.split()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_w2v = gensim.models.Word2Vec(\n",
    "            tokenized_tweet,\n",
    "            vector_size=200, # desired no. of features/independent variables \n",
    "            window=5, # context window size\n",
    "            min_count=2,\n",
    "            sg = 1, # 1 for skip-gram model\n",
    "            hs = 0,\n",
    "            negative = 10, # for negative sampling\n",
    "            workers= 2, # no.of cores\n",
    "            seed = 34)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4665180, 6285280)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_w2v.train(tokenized_tweet, total_examples= len(train['clean2']), epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('fullest', 0.48751944303512573),\n",
       " ('eliminate', 0.4714736044406891),\n",
       " ('selah', 0.4652027487754822),\n",
       " ('existence', 0.4591760039329529),\n",
       " ('hathaway', 0.45492711663246155),\n",
       " ('sober', 0.4410242736339569),\n",
       " ('shyan', 0.43827003240585327),\n",
       " ('possibility', 0.4355795681476593),\n",
       " ('emptiness', 0.43454962968826294),\n",
       " ('necessary', 0.431913286447525)]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_w2v.wv.most_similar(positive = \"life\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('linstagram', 0.09035412967205048),\n",
       " ('recordsmanagervx', 0.05600422993302345),\n",
       " ('dragoneducation', 0.0021256180480122566),\n",
       " ('rssxactaccounts', -0.018716683611273766),\n",
       " ('lozza', -0.019833624362945557),\n",
       " ('ilovethesecret', -0.026626083999872208),\n",
       " ('stamp', -0.028266310691833496),\n",
       " ('sadwav', -0.03171248361468315),\n",
       " ('oil', -0.05767695978283882),\n",
       " ('camiilabeckeer', -0.05887111648917198)]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_w2v.wv.most_similar(negative = \"hate\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier()"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "model_dt = DecisionTreeClassifier()\n",
    "model_dt.fit(x_train1, y_train1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_dt = model_dt.predict(x_test1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy : 0.9956197201984535\n",
      "Validation Accuracy : 0.9417040358744395\n",
      "f1 score : 0.5552903739061258\n",
      "[[8681  235]\n",
      " [ 324  349]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Training Accuracy :\", model_dt.score(x_train1, y_train1))\n",
    "print(\"Validation Accuracy :\", model_dt.score(x_test1, y_test1))\n",
    "\n",
    "# calculating the f1 score for the validation set\n",
    "print(\"f1 score :\", f1_score(y_test1, y_pred_dt))\n",
    "\n",
    "# confusion matrix\n",
    "cm = confusion_matrix(y_test1, y_pred_dt)\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_dt_act=model_dt.predict(X_test_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_submission = pd.DataFrame({'id':id,'label': y_pred_dt_act})\n",
    "my_submission.to_csv('twitter_6.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[19:31:26] WARNING: /opt/concourse/worker/volumes/live/7a2b9f41-3287-451b-6691-43e9a6c0910f/volume/xgboost-split_1619728204606/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=0.300000012, max_delta_step=0, max_depth=6,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=100, n_jobs=4, num_parallel_tree=1, random_state=0,\n",
       "              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n",
       "              tree_method='exact', validate_parameters=1, verbosity=None)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_xgb = XGBClassifier()\n",
    "model_xgb.fit(x_train1, y_train1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_xgb = model_xgb.predict(x_test1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy : 0.953068430697716\n",
      "Validation Accuracy : 0.9475440609031182\n",
      "f1 score : 0.44664466446644663\n",
      "[[8883   33]\n",
      " [ 470  203]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Training Accuracy :\", model_xgb.score(x_train1, y_train1))\n",
    "print(\"Validation Accuracy :\", model_xgb.score(x_test1, y_test1))\n",
    "\n",
    "# calculating the f1 score for the validation set\n",
    "print(\"f1 score :\", f1_score(y_test1, y_pred_xgb))\n",
    "\n",
    "# confusion matrix\n",
    "cm = confusion_matrix(y_test1, y_pred_xgb)\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_xgb_act=model_xgb.predict(X_test_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_submission = pd.DataFrame({'id':id,'label': y_pred_xgb_act})\n",
    "my_submission.to_csv('twitter_sentiment_xgboost.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_lr=LogisticRegression()\n",
    "model_lr.fit(x_train1,y_train1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_lr=model_lr.predict(x_test1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy : 0.9469002815894159\n",
      "Validation Accuracy : 0.9452497653561373\n"
     ]
    }
   ],
   "source": [
    "print(\"Training Accuracy :\", model_lr.score(x_train1, y_train1))\n",
    "print(\"Validation Accuracy :\", model_lr.score(x_test1, y_test1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3772241992882562"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_test1,y_pred_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[8905,   11],\n",
       "       [ 514,  159]])"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test1, y_pred_lr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_lr_act=model_lr.predict(X_test_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub=pd.DataFrame({'id':id,'label':y_pred_lr_act})\n",
    "sub.to_csv('twitter_sentiment_logisticreg.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier()"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf=RandomForestClassifier()\n",
    "\n",
    "model_rf_1 = RandomForestClassifier()\n",
    "model_rf_1.fit(x_train1, y_train1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_rf = model_rf_1.predict(x_test1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy : 0.9955750234657846\n",
      "Validation Accuracy : 0.9540098028991553\n",
      "F1 score : 0.5920444033302497\n",
      "[[8828   88]\n",
      " [ 353  320]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Training Accuracy :\", model_rf_1.score(x_train1, y_train1))\n",
    "print(\"Validation Accuracy :\", model_rf_1.score(x_test1, y_test1))\n",
    "\n",
    "# calculating the f1 score for the validation set\n",
    "print(\"F1 score :\", f1_score(y_test1, y_pred_rf))\n",
    "\n",
    "# confusion matrix\n",
    "cm = confusion_matrix(y_test1, y_pred_rf)\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_rf_act=model_rf_1.predict(X_test_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "sb=pd.DataFrame({'id':id,'label':y_pred_rf_act})\n",
    "sb.to_csv('twitter_sentiment_ranforest.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.wrappers.scikit_learn import KerasClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n",
    "def lemmatize(data):\n",
    "    lem_data = []\n",
    "    max_len = 0\n",
    "    for text in data:\n",
    "        lem_text = ''\n",
    "        text_len = 0\n",
    "        for word in text.split():\n",
    "            word = lemmatizer.lemmatize(word)\n",
    "            word = lemmatizer.lemmatize(word, pos='v')\n",
    "            lem_text = lem_text + ' ' + word\n",
    "            text_len = text_len + 1\n",
    "        lem_data.append(lem_text)\n",
    "        max_len = max(max_len, text_len)\n",
    "        \n",
    "    return lem_data, max_len\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31\n"
     ]
    }
   ],
   "source": [
    "\n",
    "X_train_lem, max_len = lemmatize(train['clean'])\n",
    "print(max_len)\n",
    "X_test_lem, _ = lemmatize(test['clean'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' when a father be dysfunctional and be so selfish he drag his kid into his dysfunction',\n",
       " ' thank for credit i cant use cause they dont offer wheelchair van in pdx',\n",
       " ' bihday your majesty',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' factsguide society now',\n",
       " ' huge fan fare and big talk before they leave chaos and pay dispute when they get there',\n",
       " ' camp tomorrow danny',\n",
       " ' the next school year be the year for exam cant think about that',\n",
       " ' we win love the land',\n",
       " ' welcome here im it so',\n",
       " ' consumer price index mom climb from previous to in may',\n",
       " ' we be so selfish',\n",
       " ' i get to see my daddy today',\n",
       " ' call middle school build the wall chant',\n",
       " ' no comment in',\n",
       " ' ouchjunior be angry',\n",
       " ' i be thankful for have a paner',\n",
       " ' retweet if you agree',\n",
       " ' it smile all around via ig user make people',\n",
       " ' a we all know essential oil be not make of chemical',\n",
       " ' people blame ha for concede goal wa it fat rooney who give away free kick know bale can hit them from there',\n",
       " ' sad little dude',\n",
       " ' product of the day happy man tool who it the time to open up &amp drink up',\n",
       " ' lumpy say i be a prove it lumpy',\n",
       " ' to my',\n",
       " ' beautiful sign by vendor for',\n",
       " ' all when be in sunday love',\n",
       " ' we have a great panel on the mediatization of the public service',\n",
       " ' happy father day',\n",
       " ' people go to nightclub to have a good night and man action mean those people be lose to their family forever',\n",
       " ' i have never have a chance to vote for a presidential candidate i wa excite about and this cycle look to be no different',\n",
       " ' doe',\n",
       " ' rip to the fellow nohern ireland fan who sadley pass away tonight gawa forever sing and cheer on fire',\n",
       " ' it wa a hard monday due to cloudy weather disable oxygen production for today',\n",
       " ' it unbelievable that in the st century wed need something like this again',\n",
       " ' bull up you will dominate your bull and you will direct it whatever you want it to do w',\n",
       " ' morning~~',\n",
       " ' once more only one word tell it all',\n",
       " ' oh wait hour in the valravn line and it stop work we be so close',\n",
       " ' i be thankful for sunshine',\n",
       " ' when you finally finish a book youve be work on for awhile',\n",
       " ' yup be a knicks fan be hard so it easier to just be an nba fan when the playoff roll around',\n",
       " ' there be life after social network embrace each day be',\n",
       " ' my mom share the same bihday a bihday snake see you this weekend',\n",
       " ' lovely echeveria bloom',\n",
       " ' i be amaze',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' im and go',\n",
       " ' feel blue',\n",
       " ' the best pa about life be know who you be',\n",
       " ' get ready remove the victums frm',\n",
       " ' for her we get her a we love you',\n",
       " ' off to concelebrate at the for the first time',\n",
       " ' let the scum baggery begin',\n",
       " ' thank you super love it zpamdelacruz dolores capas tarlac',\n",
       " ' a scourge on those play baroque piece on piano beyond belief',\n",
       " ' let fight against',\n",
       " ' happy father day mr rayos',\n",
       " ' ascot time with this babe',\n",
       " ' the weekendis here',\n",
       " ' happy at work conference right mindset lead to culture of development organization',\n",
       " ' christina grimmies last performance before be shoot via',\n",
       " ' we be ready to dance',\n",
       " ' youve really hu my feel',\n",
       " ' my wife whom i adore have to miss your poland show because she have surgery her name be bridget &amp shes my everything',\n",
       " ' i be so jealous of you right now',\n",
       " ' i celebrate every man that ha play it fatherly role father day',\n",
       " ' im sure they be just a happy hour',\n",
       " ' the white establishment cant have blk folx run around love themselves and promote our greatness',\n",
       " ' good morning the journey begin',\n",
       " ' if you like this from',\n",
       " ' our new brochure have arrive how excite',\n",
       " ' so much stuff happen in florida first shoot and now on a two year old kid',\n",
       " ' ferrari will do it for the sake of the championship this gp be clearly a turn point rb ferrarimercs',\n",
       " ' ace my first test',\n",
       " ' seek probe into leak point finger at',\n",
       " ' wrap up th',\n",
       " ' hey white people you can call people white by',\n",
       " ' you might be just have not show here today regurgitate talk point and name call',\n",
       " ' sometimes you have to raise a few brow to raise the bar',\n",
       " ' about that',\n",
       " ' design learn space to include h',\n",
       " ' how the us &amp insecurity to lure men into',\n",
       " ' carry a gun wouldnt of help if you cant take it in with you gun control wont stop the black market terrorism will get worse',\n",
       " ' use the power of your mind to your body',\n",
       " ' woohoo just over week to go',\n",
       " ' be in a far away place where you have no family member hus',\n",
       " ' ready to rehearse tonight with new music and new video look out for the announcement',\n",
       " ' now on monday night at pm on channel i finally get to see what all the fuss be about',\n",
       " ' watch the new episode of on',\n",
       " ' offline now after a very nice and long night',\n",
       " ' thing incredibly do',\n",
       " ' yes receive my acceptance letter for my master so will be back at again in october',\n",
       " ' daughter rid her bike around driveway son play his guitar for u while we enjoy by the campfire',\n",
       " ' omg love this station way to jam out at work while get work do of course',\n",
       " ' ill always hope that one day ill get to hug you but i dont think that it gonna happen anytime soon',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' couple have sex fat naked japanese girl',\n",
       " ' on that day edwardsville pennsylvania',\n",
       " ' personalise we gbp get here',\n",
       " ' there be some truly sick ppl out there',\n",
       " ' trump call on obama to resign over the orlando shoot the boy ha a point year and all he do wa talk about change',\n",
       " ' not long now',\n",
       " ' usd clear barrier jump to fresh week high',\n",
       " ' go to la tomorrow',\n",
       " ' i be thankful for good friend',\n",
       " ' i still can not wrap my head around the fact that be go and the fact that a man destroy just',\n",
       " ' just receive dis from cant wait to sta bake',\n",
       " ' we be so to be play for via',\n",
       " ' yes yes yes',\n",
       " ' sunday',\n",
       " ' im not interest in a that doesnt address &amp racism be about bring',\n",
       " ' one of my beloved long lose cd now recover thank to apple music',\n",
       " ' vine by',\n",
       " ' why not mock obama for be black',\n",
       " ' the spell of brexit referendum commerzbank',\n",
       " ' amaze health benefit of cucumber be',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' off to work',\n",
       " ' i make it',\n",
       " ' lap of pool k ride do and pick up a gym membership form',\n",
       " ' polar bear climb race angry polar bear climb race the polar bear live in cold place look',\n",
       " ' get him angry a muslim assassinate not so much',\n",
       " ' happy snappy wait for the football',\n",
       " ' friday i miss you',\n",
       " ' la vega strip',\n",
       " ' ma fleurette',\n",
       " ' sad in the branch it just rainy day write tear be fly bird',\n",
       " ' yeah new button in the mail for me they be so pretty',\n",
       " ' driver hit female moose on river rd moose wa kill driver be ok crew remove animal now',\n",
       " ' afterpas make in japan madeinjapan cute',\n",
       " ' arent protest because a win they do so because trump ha fuhered &amp',\n",
       " ' i need to find a way to spend my time so you be not always on my mind',\n",
       " ' update',\n",
       " ' anyone know the date our and become corrupt',\n",
       " ' make me more than even why put a installation if u be go to take it off',\n",
       " ' bihday',\n",
       " ' d most impoant thing be to your life to be it all that matter life be too sho',\n",
       " ' happy bihday chris evans a great actor and human',\n",
       " ' our heas think prayer go out to the more than people who be murder a gay nightclub in',\n",
       " ' demo guitar for new album',\n",
       " ' retweeted lion pro',\n",
       " ' usd target the week sma at',\n",
       " ' ive have pretty bad bihday week before but so far this be the worst ever',\n",
       " ' so bless to have work with sa best lead lady',\n",
       " ' happiest place on eah',\n",
       " ' be kinda to be among human again',\n",
       " ' just find out be at after id be exclaim to how gr blain hair wa the other nite',\n",
       " ' &lt listen to my most beautiful best friend sing with her most amaze voice &lt &lt &lt &lt',\n",
       " ' be n freedom from effo in the present merely mean that there ha be effo store up in the past theodore roosevelt',\n",
       " ' when you know yall aint go know where',\n",
       " ' yes it when you call a gorilla because racist have long think of black people a no bet',\n",
       " ' this be u all this year wait for the show to sta our rd year run',\n",
       " ' when the internet be break so you cant watch netflix',\n",
       " ' and she will do just thatthen whatbusiness a usual',\n",
       " ' hbd to this dick suckin tequila lovin slut i wouldnt want u any other way',\n",
       " ' a the smaller hand show barry probably lie about be why his game suck more than his',\n",
       " ' good morning friday what be everyone do this weekend',\n",
       " ' challenge s claim that be dare it to prove',\n",
       " ' send my deepest condolence to the orlando gay family zimbabwe gay community ha you at hea',\n",
       " ' new selfie \\\\',\n",
       " ' saturday afternoon chi meet up',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' soed out all my nut &amp bolt',\n",
       " ' aww yeah it all good bing bong bing bong',\n",
       " ' just dawn on me in month i will be see live i be so excite',\n",
       " ' happy th bihday shane robe watson',\n",
       " ' you point one finger million be point right back at you',\n",
       " ' sad to hear the announcer say that it may have move the player to one or two out of the lead',\n",
       " ' im go to tonight oh yes',\n",
       " ' u think like this too the god who be not come through for me the god who be not keep me',\n",
       " ' terrorist = constitutional right this be just another excuse by republican to appease the nra with no gun control measure',\n",
       " ' it all about',\n",
       " ' very sad what happen',\n",
       " ' everyone be everything',\n",
       " ' perhapse a good example of the need to protect from within europe',\n",
       " ' angry bird stella bad princess birdsstellabadprincess time post wed',\n",
       " ' check incredibly to have make great memory with great people we do an',\n",
       " ' saw a t shi last night that say rip to when i care',\n",
       " ' hilarious im already block by for ask one not paicularly difficult question',\n",
       " ' devastate news my hea go out to the victim and their family',\n",
       " ' happiness be not a state to arrive at but a manner of travel margaret lee',\n",
       " ' safe way to heal your',\n",
       " ' what i be create right now',\n",
       " ' wed number',\n",
       " ' be look forward to attend the cipd ireland conference &amp workshop',\n",
       " ' finally reach follower on follower hypu tomorrow',\n",
       " ' word r free it how u use em that can cost you',\n",
       " ' after what our country go through this last year to see people bring a bomb to the stadium &amp come here for everything but football',\n",
       " ' i be thankful for cat',\n",
       " ' time to eat with my bae swalscha',\n",
       " ' ilovethesecret',\n",
       " ' you might be a libtard if',\n",
       " ' it about to go down',\n",
       " ' attack bull game d do you really think that his head wa empty around the city each side',\n",
       " ' what a self serve hypocrite always keep your eye on the bounce ball with this nimrod president',\n",
       " ' bull up you will dominate your bull and you will direct it whatever you want it to do whe',\n",
       " ' polar bear climb race angry polar bear climb race the polar bear live in cold place loo',\n",
       " ' if they want reelection money',\n",
       " ' my hea be with right now i truly cannot imagine could have easily happen anywhere',\n",
       " ' alreemgroup',\n",
       " ' it seem like the only place with action here be montreal',\n",
       " ' that and you be old and wash up',\n",
       " ' i be spirit',\n",
       " ' i be thankful for enteainment',\n",
       " ' this show truly how bad brisbane be',\n",
       " ' cant wait for',\n",
       " ' happy man',\n",
       " ' moment the lucky chinatown mall binondo',\n",
       " ' this really take the piss im so angry just go to show who be value and who isnt you complete and utter moron',\n",
       " ' take out the america i vote against i vote against i vote against i vot',\n",
       " ' father day to all the out there that give up so much for their family your',\n",
       " ' doe really work',\n",
       " ' sebbo ha say it all',\n",
       " ' eg smile',\n",
       " ' brilliant service at your kettering branch today customer',\n",
       " ' will be here for a screen i will miss it so i will sing all the song from potp and shock treatment because i can',\n",
       " ' on the way toraya',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' good day new beer at bridpo food &amp beer fest back at the campsite drink the starstruck &amp move on to',\n",
       " ' g o a l s repost from sydney',\n",
       " ' i want to teach you love like youve never felt it before',\n",
       " ' i be thankful for now',\n",
       " ' omg just day leave cant wait for him to come to milan',\n",
       " ' stone rise do now time for in a d club day',\n",
       " ' there a reason why two people stay together they give each other something nobody else can hopelly great day',\n",
       " ' you never know who might have a crush on you',\n",
       " ' my friend just get engage',\n",
       " ' day lef b end of classez yay',\n",
       " ' may we watch how we say what need to be be say',\n",
       " ' i be legendary',\n",
       " ' to the unite state of america from the rest of the world i think youve prove you dont have the right to own firearm',\n",
       " ' if you hold open a door for a woman because shes a woman and not because it a nice thing to do thats dont even try to deny it',\n",
       " ' when your have a good weekend and it show',\n",
       " ' when you have a good hea and help everyone you always seem to be the one who get hu',\n",
       " ' the happiest baby ive ever know',\n",
       " ' be you ready for bc be',\n",
       " ' cant wait for carowinds ready to see and time to',\n",
       " ' already buy my find dory ticket',\n",
       " ' for every minute you be you lose sixty second of and you should do the same',\n",
       " ' see you guy in september',\n",
       " ' question of the day excite for motos',\n",
       " ' make today count',\n",
       " ' happy morning degree with anna',\n",
       " ' would u be able to suppo our event they lose fund and may have to close',\n",
       " ' we be happy little people happyhappy people',\n",
       " ' thank you very much for this oppounity will cherish it forever',\n",
       " ' themeowood puppy',\n",
       " ' few more day till who be',\n",
       " ' happy by pharrell williams lyric',\n",
       " ' happy at work conference right mindset lead to culture of development organization',\n",
       " ' we lose another member of our family yesterday',\n",
       " ' yayyy trailer morrow super',\n",
       " ' watch fancy tail vine mad',\n",
       " ' bull up you will dominate your bull and you will direct it whatever you want it to do when',\n",
       " ' im so ready for tomorrow',\n",
       " ' in memory of my wonderful dad always miss eternally love',\n",
       " ' on dallas shoot the video youre about to see be disturb so viewer discretion then they play it in a loop ove',\n",
       " ' s fine a little',\n",
       " ' sunshine &amp sticker kind of saturday',\n",
       " ' have a wonderful day today',\n",
       " ' buka bersama tma family w risha meylucky &amp others at mayfair townhouse pic',\n",
       " ' he love you more than you know',\n",
       " ' this man run for governor of ny the state with the biggest african american population',\n",
       " ' offer no or solution but create the same old repetitive',\n",
       " ' who can get the pussy quicker these day',\n",
       " ' ill probably be by the time come back home',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' anyone play voez kinda in love with this game',\n",
       " ' how many pass by how many time and say nothing',\n",
       " ' sequoia be about the',\n",
       " ' some bitch care more about their eyebrow then how much they makin at the end of the week',\n",
       " ' bham be pride paradeready',\n",
       " ' to all the guy who play dad out there we you',\n",
       " ' bihday to my bogummylove you foreverhope you happy everydaygood night',\n",
       " ' wish all our client have baby goat to cuddle',\n",
       " ' love you guy get some concord mill mall',\n",
       " ' the get ready to be scar tonight it on',\n",
       " ' today be the day to make the change eliminate negativity and choose to live a positive lifestyle instead',\n",
       " ' aaahh nabilah look so arigatou jkt have a very happy daaay',\n",
       " ' last day at work tomorrow for me self employ from monday',\n",
       " ' im half boy half man so im dumb too',\n",
       " ' my happy little boy',\n",
       " ' never be more appropriate i have zero idea of what im feel right now',\n",
       " ' have a magnificent monday',\n",
       " ' i just dont understand y these designer print animal on fabric just buy muzlin b',\n",
       " ' maybe happy hour you can get a',\n",
       " ' and her friend',\n",
       " ' if you have never lose a love one to senseless violence you dont get a fuck opinion on the nd amendment',\n",
       " ' where be the world head you could lose your life just for leave your home',\n",
       " ' i hear him say that he wa try to lock the shooter in but keep other patron from get out',\n",
       " ' good mood flight with off to munich for interview',\n",
       " ' first presentation by in the netherlands by',\n",
       " ' u make u proud i hv be read ur word &amp assure it will b a assoment of petite emotion wish u the best',\n",
       " ' happiness v joy happiness be temporary and leaf u hu joy last forever and never leaf',\n",
       " ' twinklatinboys na',\n",
       " ' it be not people who be thankful it be thankful people who be happy',\n",
       " ' i be creative',\n",
       " ' i have hold back tear i have cry happy tear &amp finally we leave a week tomorrow',\n",
       " ' bihday pressie from my mummy and my granny',\n",
       " ' stick in athens instead of santorini because say it wa to windy to land shock service on every level since',\n",
       " ' these happy folk have the very first at the launch',\n",
       " ' what a huge crowd for trump',\n",
       " ' i be thankful for saturday',\n",
       " ' good god why',\n",
       " ' enjoy these delicious melon',\n",
       " ' first month then week now it only day till the begin of the end of i be both and at the same time',\n",
       " ' it a firework weeheeeee~',\n",
       " ' attack bull game d do you really think that his head wa empty around the city each side be',\n",
       " ' to all our the amaze father out there today',\n",
       " ' i just want to be be that too much to ask for',\n",
       " ' ready and wait',\n",
       " ' im sad my aunt pass away she wa',\n",
       " ' horribly ad from that make u cringe',\n",
       " ' wouldnt mind robson kanu at s decent player available on a free transfer too',\n",
       " ' come on you can do better than this please put yourself in s shoe unnecessary stress h',\n",
       " ' stalk me watch me im just a peon poor disable kill methis be not the usa i use b proud of &amp the universe doesnt care',\n",
       " ' s outburst beg the question can christian get',\n",
       " ' im so and now that',\n",
       " ' more week to go then im san antonio bind to see i cant wait to see my little brother',\n",
       " ' listen up &gt&gt&gt&gt&gt&gt&gt',\n",
       " ' the die of the light village green town',\n",
       " ' hahaha this be me last &amp',\n",
       " ' hey i notice that u be a train lover so i ask u to check my free app it know a my train on apple',\n",
       " ' sexy a fuck',\n",
       " ' sday say i love you to your dad right now',\n",
       " ' guess who back back again',\n",
       " ' hey superspoilt you may want to check for latest update on',\n",
       " ' bihday sid god bless you hav a successful career',\n",
       " ' nba final = ambivalent love steph and thompson kyrie stink it up green be an class a doosh lebron wont talk to guillermo',\n",
       " ' check out my production team web series episode that wa edit by yours truly',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' you never answer me about your statement on the s over &amp be find w',\n",
       " ' independence hall',\n",
       " ' gorilla simulator you need to do to adapt to the environment the need to tear the city material',\n",
       " ' watch the leadership embrace &amp kiss donalds as be &amp this be now the pay a pay of racism &amp hate',\n",
       " ' i love how i be so mature for my age',\n",
       " ' aww yeah it all good bing bong bing bong',\n",
       " ' it such a good feel when this happen',\n",
       " ' check it',\n",
       " ' in day aaahhhhh',\n",
       " ' it today ht',\n",
       " ' sun be out happy thursday in the',\n",
       " ' pack sneak peek to be work with the of the for',\n",
       " ' tony sunday',\n",
       " ' happy father day to my dad and all of the dad out there',\n",
       " ' theverdict be in im an idiot',\n",
       " ' of the week june',\n",
       " '',\n",
       " ' my little brother ha a higher tolerance of alcohol than me lol',\n",
       " ' wish this guy a blaze say be the spit image of a',\n",
       " ' i be thankful for hike trail',\n",
       " ' all pay meet to tackle a movie which uncover a problem we dont want the world to know',\n",
       " ' chill relax piano music to keep your pet and during',\n",
       " ' you might be a libtard if',\n",
       " ' nothing say merica like that tweet',\n",
       " ' always be',\n",
       " ' today i lose a sister senselessly we sing for love not death people',\n",
       " ' some guy too concern about their twitter brand to be honest with their follower',\n",
       " ' i hate like of my family i wa really bear into a big as toxic bomb mother fucker dont give a fuck about me cuz i get money',\n",
       " ' get get get',\n",
       " ' best for be',\n",
       " ' trump ny co chair make racist remark about michelle obama',\n",
       " ' be you a fan this should make you',\n",
       " ' a a a tool for be',\n",
       " ' how can food make a family fight mxm',\n",
       " ' delicious sangria the pub pembroke',\n",
       " ' have a fabulous weekend beautiful soul',\n",
       " ' when im feel and i look down and see a million reason to be again i love you craig',\n",
       " ' back to the day job for shane lowry',\n",
       " ' hour right now bottle &amp drink from pm w chippewa st',\n",
       " ' ana + nimra + farishta + mawia + khushal + + + + = the number of mark obtain be too damn high',\n",
       " ' my monday be so empty without the bachelorette',\n",
       " ' suppoing family god chivalry tradition and pride in one heritage be now controversial',\n",
       " ' the pic say otherwise for young girl confine in that kitchen you be void of mean beyond cheap publicity',\n",
       " ' night ever',\n",
       " ' do what you love to do simply for the love of it',\n",
       " ' lunchtime with mom',\n",
       " ' my sister aint even know i wa watch her argue with the nigga wa wait for him too say the wrong shit',\n",
       " ' ride out ride back do',\n",
       " ' well i wa gonna go see adtr with bae',\n",
       " '',\n",
       " ' polar bear climb race angry polar bear climb race the polar bear live in cold place',\n",
       " ' healthy fam',\n",
       " ' dream palace can we move in already day',\n",
       " ' im wait for a follow since the x factor',\n",
       " ' but it your fault you have to use it to pointscore',\n",
       " ' this week staed off kinda lame but it get better',\n",
       " ' young man he luks too hot wen he be in temper ill b alwys ur fan till my lst breath h',\n",
       " ' dont worry',\n",
       " ' double tap and follow me',\n",
       " ' for be',\n",
       " ' turn flat to eye eia inventory data for fuher direction',\n",
       " ' train in new employee be fun',\n",
       " ' to our',\n",
       " ' pig burn alive in saskatchewan barn fire video',\n",
       " ' youll never convince me that american have the right to bear these',\n",
       " ' with my beautiful lady be a bless to know them',\n",
       " ' today baba siddique ifftar pay at mumbai &amp dono ayenge to see both again',\n",
       " ' hand',\n",
       " ' i cant wait to go see warcraft ugh',\n",
       " ' cousin reunite wellness',\n",
       " ' at for an audience with duane henry should be very insightful',\n",
       " ' my train hasnt move so im binge tweet',\n",
       " ' have my lover stop be angry at me visit us&gt&gt&gt',\n",
       " ' when youre block by a troll because you promise to &amp let his nonsensical rant boo',\n",
       " ' father day to all the dad here and the one that be in heaven',\n",
       " ' june',\n",
       " ' how creepy awesome be these baboon climb the fire escape like human on',\n",
       " ' use the power of your mind to your body be',\n",
       " ' dude im a tiger',\n",
       " ' we dont because be happy be because we laugh william jam',\n",
       " ' have a happy sunday everyone i think today be go to be a good day',\n",
       " ' can we talk about the way in which organization dehumanize u and how theyre never hold accountable',\n",
       " ' stick in the rat race find your freedom',\n",
       " ' i have to study i steal my roommate idea for take this pic',\n",
       " ' dinner with sister',\n",
       " ' who else be plan on watch tomorrow',\n",
       " ' rifle day snow out so ar be closest to m i ever get',\n",
       " ' euro bbc pundit make their prediction for france sta today',\n",
       " ' because i be happy',\n",
       " ' no elder scroll for many year wah wah',\n",
       " ' poor and a little bite cheeky &amp',\n",
       " ' cannot wait for this year',\n",
       " ' very to work with russellbeckwith soon my friend lyle',\n",
       " ' why the nazi study american race law for inspiration',\n",
       " ' share ice cream',\n",
       " ' nothing like a river island sale shop spree to cheer you up',\n",
       " ' clean ze house and zoing ziy laundry im finally free from fever and cold im happy',\n",
       " ' new list be up',\n",
       " ' i be thankful for my challenge',\n",
       " ' chill night in with ice cream&amp grow up',\n",
       " ' absolutely originate in hateful hatred of our african american',\n",
       " ' * sweet *',\n",
       " ' all these tragedy in america first with the kill of and now be not a safe place to be today',\n",
       " ' angst a factor help drive down interest rate in germany and japan bbh',\n",
       " ' fab &amp interest day at now dash to a quiet place to dial into iiba emea region meet',\n",
       " ' kayak sup snorkel swimwhatever your pleasure well put it together',\n",
       " ' u can hold a man down for year and watch him change on u over some hear say no loyalty no',\n",
       " ' well back go to tacloban now see ya later travel to tacloban city from catbalogan',\n",
       " ' how do you feel about that you havent win any olympics medal',\n",
       " ' angry bird il film download ita p hdtc',\n",
       " ' to you all by',\n",
       " ' fo woh officer in viral arrest video wa rude not racist chief',\n",
       " ' saturday friend',\n",
       " ' yayyyy my friend get marry today congratulation guy',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' if lewis remain perfect the ranger announcer be gonna stroke out',\n",
       " ' chris be so to be back to after miss most of with a break',\n",
       " ' train ticket book for my opera week in wale',\n",
       " ' it a beautiful day beach park',\n",
       " ' the big screen be be fit right now',\n",
       " ' not all love story be happy end',\n",
       " ' bull up you will dominate your bull and you will direct it whatever you want it to do when you sta',\n",
       " ' i be tranquil',\n",
       " ' be that the name of any upcoming new track',\n",
       " ' best for',\n",
       " ' wow some people child have nothing better to do then insult people on twitter',\n",
       " ' think maybe i should have buy instead of',\n",
       " ' the fact that album be an hour away from drip ha me lose my mind',\n",
       " ' ryderrideu see you cc and cum together',\n",
       " ' snapchat gabychina',\n",
       " ' i be brilliant',\n",
       " ' true nigga dont go look for apaments they go look at shoe and jewelry',\n",
       " ' safe way to heal your',\n",
       " ' u r so special u father day',\n",
       " ' camp next weekend',\n",
       " ' im so and now that',\n",
       " ' my bestie',\n",
       " ' absolutely love this place fantastic welcome from carlos receptionist th time here room request grant a usual',\n",
       " ' head to with my peculiar family',\n",
       " ' retail sale control group likely rise m m in may danske bank',\n",
       " ' happy father day you be love of my life &ampthe best father in the world',\n",
       " ' coolestlifehack world smallest snake',\n",
       " ' this be a shockingly and increasingly we live in and the ha gun control law i feel for the mp family &amp friend',\n",
       " ' or simply so',\n",
       " ' happy bihday to me brithday day holiday',\n",
       " ' i be sadden that no one ha think to groom her for the position before ask her opinion on such an issue',\n",
       " ' ukrainian athlete be not in grosseto for watch livestream without see them it not the same',\n",
       " ' new season of oitnb stas tonight',\n",
       " ' excite to be on the other side of the cuain for once tonight look forward to see on broadway',\n",
       " ' when quay collab with say sell out',\n",
       " ' im to finally get my and i get to them too',\n",
       " ' i be impressive',\n",
       " ' father day to all the father out there out there make it happen to yall',\n",
       " ' our view from the th floor girl',\n",
       " ' discuss republican be the same thing a critique we cant ignore the consequence',\n",
       " ' get school tomorrow my last integration',\n",
       " ' amen truth always &lt &lt',\n",
       " ' and the stamford kid get month for rapebut that wasnt boredomit wa minute of action',\n",
       " ' let colorize your life w rainbow depok',\n",
       " ' rude customer care executive make me want to switch my loyalty to another provider',\n",
       " ' hi teddy corpuz',\n",
       " ' wow what a great daughter you be to send that to your father right before father day',\n",
       " ' reason to be happywhen your kid be mid tantrum',\n",
       " ' i be truthful',\n",
       " ' yup on woot next do they work on her listen skill',\n",
       " ' do brockely london',\n",
       " ' be you &amp feel like the be stomp on you',\n",
       " ' thats the problem only one police officer you need more with bigger gun and perhaps one tank with soldier',\n",
       " ' father day to all friend',\n",
       " ' if youre not with your this fall you can write a candidate in',\n",
       " ' end',\n",
       " ' get to know the team behind the thats keep our tummy in our',\n",
       " ' actually go fish today somewhere nice',\n",
       " ' how cute they both look like doll',\n",
       " ' i do and im',\n",
       " ' love this vacation',\n",
       " ' unreal advisor refuse to take my payment and say wait for pcn be this customer service when ur site wasnt work',\n",
       " ' im for forza horizon on the pc what a time we live in',\n",
       " ' life better in tahiti',\n",
       " ' i know their male but i think they have a vagina like in video game xd',\n",
       " ' pa of the pack wa take and brainwash by cheetah',\n",
       " ' domestic corporate good price index yoy in line with expectation in may',\n",
       " ' remember the day when people on go on to become reality tv star no it seem reality tv star become big brother contestant',\n",
       " ' the more familiar we become the more difficult it be to stereotype',\n",
       " ' if you want to live a life tie it to a goal not to people or thing',\n",
       " ' friday &lt',\n",
       " ' day porn movie pussy pay pic',\n",
       " ' it look easy till my turn',\n",
       " ' banner and newyear',\n",
       " ' yay thank for let me know my pa will be a very happy man',\n",
       " ' in my exam room',\n",
       " ' best homeopathic remedy for',\n",
       " ' rest in peace christina',\n",
       " ' those bamas you fool around with might be raise your child ppl dont think that far in advance they just see fun or whatever',\n",
       " ' anti racist be a codeword for antiwhite asia for the asian africa for the african white country for everyon',\n",
       " ' on my first god baby graduate from high school',\n",
       " ' nearly there folk just finalise my stuff with a few pointer from grainne',\n",
       " ' fellowship',\n",
       " ' i just catch myself eat chocolate slice bread beside the ref realize i still didnt eat my dinner',\n",
       " ' you police during the police be under a consent decree be',\n",
       " ' best for be',\n",
       " ' everything you need for to all the out there from',\n",
       " ' agree',\n",
       " ' our next show be monday at the fiddler elbow in camden it look like it go to be a huge night ht',\n",
       " ' no swim sign everywhere parent werent watch a year old',\n",
       " ' mitchellrattle mitchell goodson',\n",
       " ' get ready for saturday jeep run with nbjc day until htt',\n",
       " ' attempt at czech fury only for croatia to score their second',\n",
       " ' the latest dairy k thank to',\n",
       " ' yep grateful for everyday',\n",
       " ' ive have the ebook for awhile but it so nice to have the hard copy',\n",
       " ' nba final game what more could a basketball fan ask for',\n",
       " ' that doesnt prevent you from be a and a',\n",
       " ' dear despite our attempt to i regret to announce that weve lose her angelic friend we will miss u',\n",
       " ' place color or use u',\n",
       " ' perfect moment of the day',\n",
       " ' wait for for an hour to come out just for an autograph finally walk awayhe come out of course disappoint',\n",
       " ' gift your a secure',\n",
       " ' girl in the world sexy mature housewife',\n",
       " ' time to get happy it saturday',\n",
       " ' to my awesome dad my almighty father my lord my darling dad rip sweethea my husband my baby daddy st son',\n",
       " ' owen football tornament',\n",
       " ' come on piss poor reception today a month and struggle to get a signal in put more mast up or im switch',\n",
       " ' excite na me',\n",
       " ' let begin we be very bihdayyourmajesty ht',\n",
       " ' wolf in last stage now',\n",
       " ' i dont like the slogan talk about be',\n",
       " ' rip little bear bear',\n",
       " ' give me a hug my cat moment',\n",
       " ' your lip like fire my blood like gasoline',\n",
       " ' here we go let do this people',\n",
       " ' life isnt a race allow yourself to be in the present',\n",
       " ' dental testimonial from our client',\n",
       " ' thank yousoo much pretty i feel soo bless &amp luckyplz do keep shower ur lyks on me',\n",
       " ' nice c susanna smile daywho need false promise of fly dragon when shes get ben',\n",
       " ' d not back yet',\n",
       " ' i get the call yesterday mom wa diagnose with stage ovarian cancer im terrify shes terrify just dont know what to do',\n",
       " ' why be ignore',\n",
       " ' be on the other side of history and our ally on this issue',\n",
       " ' fox new just come out and say it bluntly one of the most thing ive read',\n",
       " ' good morning all',\n",
       " ' monkey be also declare a vermin no protest from bajrang dal',\n",
       " ' new plant from be here and ready for the garden at garden know how',\n",
       " ' coldplay last night with my fave',\n",
       " ' use the power of your mind to your body',\n",
       " ' here mine in this gorgeous sky',\n",
       " ' aww yeah it all good bing bong bing bong',\n",
       " ' yes let do thissuppoing a openly who advocate more of same',\n",
       " ' pixion wallpaper image a',\n",
       " ' good moooooorning',\n",
       " ' enjoy everything already give to you rather than always need more',\n",
       " ' match schedule design france v romania all feedback be appreciate',\n",
       " ' for father day too should take your dad to for the for he will love it',\n",
       " ' berlin before and after thank you so much that wa incredible',\n",
       " ' why listen incubus never hus nissan de',\n",
       " ' anderson cooper &amp don lemon will not like gay be throw off build by sharia',\n",
       " ' bei be on a field for dog walk',\n",
       " ' who would steal awork from',\n",
       " ' compete for a list today',\n",
       " ' great park but a totally disorganise one',\n",
       " ' ive get news for you bernie wasnt win dc even if the vote wa hold last month',\n",
       " ' usd print new monthly top above draghi eye',\n",
       " ' our two presidential candidate everybody this be what weve devolve into',\n",
       " ' poor pup vine by',\n",
       " ' rare pic of superstar with bihday sivakoratala',\n",
       " ' and how be it and i be move just outside of downtown gr mi and neither of you offer better than mbps',\n",
       " ' wish all the dad a happy father day',\n",
       " ' meet this girl yr ago online b ng gay friend y doe she look like this when she didnt b',\n",
       " ' that face say it all',\n",
       " ' b u s y',\n",
       " ' enjoy this weekend with your family at your home happy weekend my dear friend',\n",
       " ' another senseless tragedy',\n",
       " ' love frm the bottom of my hea frm the day it star &amp what do v get in return',\n",
       " ' captain felix at rutland water the little boat behind be about to go on a very grand adventure',\n",
       " ' binge watch series of so a to completely avoid football',\n",
       " ' to quote jeremy corbyn here we here we here we fuck go',\n",
       " ' happy',\n",
       " ' porn vids wwwsmallgirlsexcom',\n",
       " ' to do what make your',\n",
       " ' home sweet home be what i call since move there year ago',\n",
       " ' play england v russia in marseille at what do they expect high risk game and plenty of time to drink',\n",
       " ' aww yeah it all good bing bong bing bong',\n",
       " ' i be thankful for sweater weather',\n",
       " ' haunt atmospherics',\n",
       " ' become a fan doe not give the right to criticize the personal life of an idol',\n",
       " ' the latest obsidian radio daily thank to',\n",
       " ' mind blow learn so much in this google form workshop',\n",
       " ' so sexy',\n",
       " ' back to after a v traumatic few month damage not a bad a think proof the plan work onwards &amp upwards',\n",
       " ' you might be a libtard if',\n",
       " ' just more overwhelm evidence that the company keep echo his sentiment &amp ideal',\n",
       " ' most of yall female beef out over these community dick as nigga',\n",
       " ' people be like you already forget so and so nah it not that it just new name be make a hashtag on a weekly basis',\n",
       " ' how to be i have the answer',\n",
       " ' cherry color be symbol of warm and positive energy necklace price aed',\n",
       " ' i cant wait for the season to begin',\n",
       " ' happy father day to all dad except',\n",
       " ' independent brexit poll show leave vote ahead gbp usd slump',\n",
       " ' mm + mm = no washout no flood good take and all set for another day golf',\n",
       " ' ya ever since zayne leave theyve never be the same',\n",
       " ' happy father day day',\n",
       " ' big shock on monday need to find new home yeah find my dream cottage on tuesday',\n",
       " ' save thousand free search x logins x broker',\n",
       " ' actually order a kylie lip kit',\n",
       " ' i hope the family make arrangement to make sure that cant get access',\n",
       " ' u firm to build nuclear reactor in india',\n",
       " ' bull up you will dominate your bull and you will direct it whatever you want it to do when y',\n",
       " ' no matter where you go',\n",
       " ' oscar tabarez talk luis suarezs angry reaction',\n",
       " ' now play on u uk eur join u on',\n",
       " ' only week to go and im a full time i will have new slot available from june pls spread t',\n",
       " ' a mother can take care of ten childrens but sometime ten child cant take care of their mother',\n",
       " ' i be thankful for good night sleep',\n",
       " ' now play the bride nocturne on',\n",
       " ' father day to all tatay daddy enjoy wathing tha voice kid',\n",
       " ' i be thankful for my friend',\n",
       " ' i be thankful for mason jar',\n",
       " ' nigger of a white kid who grow up in the s',\n",
       " ' congrats for',\n",
       " ' via jts',\n",
       " ' ready for at',\n",
       " ' how utterly people with incurable in albino kill in malawi spark education campaign',\n",
       " ' off high back around',\n",
       " ' shock event in orlando now will the usa review their gun law or do more innocent have to die first',\n",
       " ' i be knowledgeable',\n",
       " ' with le sister',\n",
       " ' feel so and',\n",
       " ' tag be the saddest representation of how people feel they be now force to vote for her',\n",
       " ' the h factor doe your app make the user want more factor click to download aricent',\n",
       " ' meanwhile google violate free speech at will',\n",
       " ' happy at work conference right mindset lead to culture of development organization',\n",
       " ' attack bull chase when you leave the lot despite the fact that youre a strong source of f',\n",
       " ' im in the competition',\n",
       " ' gorilla simulator you need to do to adapt to the environment the need to tear the city m',\n",
       " ' ima go cry for a few hour brb',\n",
       " ' i my sorrow to the so that the river teach me how to flow without the sepatu dahlan',\n",
       " ' i be thankful for memory',\n",
       " ' you surely be a nasty piece of work lmao',\n",
       " ' today be a perfect day to be',\n",
       " ' after sex over sex nake woman',\n",
       " ' i be thankful for cocktail',\n",
       " ' dont wait there',\n",
       " ' guess no one want to let the resident of spring street know they wouldnt be able to leave for a few hour today',\n",
       " ' great morning deborah and day',\n",
       " ' split screen',\n",
       " ' people use tragedy like this to fuel their hate agenda towards entire group of people and thats the opposite of helpful',\n",
       " ' taeyeon unnie be on fire btw i really hope that her album cover be not yellow lol',\n",
       " ' john micahow the hell be people like you elect',\n",
       " ' nothing make dad happier than a tasty meal',\n",
       " ' the about rate in young',\n",
       " ' happy sunday',\n",
       " ' oh she block me at least i have love island to look forward to on sunday',\n",
       " ' now play suzanne ciani oclock in the morning on music song',\n",
       " ' swing',\n",
       " ' help bubble attack',\n",
       " ' i be move',\n",
       " ' the thing be no matter how many law and regulation there be with gun people will still find a way to get their hand on them',\n",
       " ' quality print &amp card at',\n",
       " ' didnt stay for long have an emergency at home my daughter wa not well soo i have to rush home and i miss ur set',\n",
       " ' great insight on trust profession in emea at at',\n",
       " ' carolyn cooper ugly poor ignorant and black',\n",
       " ' franklin be a in the',\n",
       " ' perfect day= buy real furniture hike for dinner for desse',\n",
       " ' happy face lovely smile',\n",
       " ' love you',\n",
       " ' bull hill climb you have to reach the target to complete the task to survive with str',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' orlando gay men angry they can&ampt donate blood after massacre',\n",
       " ' golinglobal golinuntern fresh air trail run and a soul keep the life balance',\n",
       " ' monroe&ampnick best friend &amp rosalee&ampaddie and now kelly&ampbabymonrosale family',\n",
       " ' or lie hillary would call it a security inquiry liberal be like mind if one eat shit then all of them eat shit',\n",
       " ' you know what i find incredibly take a personal call during a meet',\n",
       " ' date night billy on burnet',\n",
       " ' i wish would make their phone so the damn screen dont shatter they cost an arm and a leg and still gotta pay to fix em',\n",
       " ' this be so hea wrench what would be the state of the mother when she know her son be about to die',\n",
       " ' how time fly le than a week to go until the brilliant lead our summer conference',\n",
       " ' fruit time',\n",
       " ' so sad one &gt&gt&gt',\n",
       " ' sirf ashiq log follow karen ki full poetry + add + follow',\n",
       " ' minute to go',\n",
       " ' thank god for the of because without it kansa city be right down there with',\n",
       " ' will get to see my favorite person tmrrw',\n",
       " ' buffalo simulation buffalo for you to take in the vicinity of their home to do in this way you',\n",
       " ' what a world we live in',\n",
       " ' now the vacation ha begin htt',\n",
       " ' a bite of heabreak too',\n",
       " ' peaceful way to sta the morning before the daily chaos begin',\n",
       " ' hardcore girl video birday sex xxx',\n",
       " ' book order arrive in time for a long weekend away in the bush',\n",
       " ' bull up you will dominate your bull and you will direct it whatever you want it to do when',\n",
       " ' steph san kana your s be badly need please',\n",
       " ' another hand &amp foot complete berachah you be the best &amptired',\n",
       " ' on my way over ctf to have a book talk with the inmate paicipating in the book club program',\n",
       " ' ready for this',\n",
       " ' negative external environment remain suppoive mufg',\n",
       " ' mkf collection jasmine and moira crossbody bag wednesday',\n",
       " ' currently out to eat at and ive be here for minute and not get the shake i order',\n",
       " ' gray be a very sad color it make me feel',\n",
       " ' difficult road often lead to beautiful destination anon',\n",
       " ' long position add eur shos increase anz',\n",
       " ' it on for me it day',\n",
       " ' no he wa definitely a mexican',\n",
       " ' when be the new album be release about and your break up',\n",
       " ' stand up to racism and hate by americad not curse out congress and all hater who be deplorable',\n",
       " ' you be my my only you make me when sky be gray',\n",
       " ' something i want to say something that make me happy',\n",
       " ' the football be actually the only thing on at mo thats watchable',\n",
       " ' happy hump day',\n",
       " ' let me just say i love be me',\n",
       " ' dont worry be happy via',\n",
       " ' because great thing never come from comfo zone',\n",
       " ' happy puppy',\n",
       " ' i dont know where youre go but do you have a room for one more trouble soul',\n",
       " ' we recommend these dark &amp',\n",
       " ' use and to win the white house press attack on khan family and gop leader',\n",
       " ' one of my bucket list item tick tonight i finally saw ice cube live tonight today wa a good day',\n",
       " ' my friend magical potion ewa beach house',\n",
       " ' everytime i wear soccer shis joie fry me and say i look mexican a fuck',\n",
       " ' it be a weird few month but finally feel more positive about life lately',\n",
       " ' check in',\n",
       " ' night a sweet dream',\n",
       " ' nyc in seven day',\n",
       " ' life be what you make it let make it',\n",
       " ' rooster simulation i want to climb the vast expanse of mountain it reach the leakage coc',\n",
       " ' in the best mood ever',\n",
       " ' be happy dont worry',\n",
       " '',\n",
       " ' just buy launch use design thinkingby &amp via',\n",
       " ' announce more office bearer of zone',\n",
       " ' so mad that i choose dj snake over zedd at edc',\n",
       " ' mango wood lett gbp get here',\n",
       " ' the danger of white liberalism to oveurning apply to liberal men and feminism too',\n",
       " ' truly truly pathetic in so many way nothing learn from the coverin',\n",
       " ' monday morning work by the lake',\n",
       " ' im love this rain hope it hang around a while',\n",
       " ' you have choose the wrong horse and unless you change your mind soon it will be the end of your political career',\n",
       " ' wed be impoant because they celebrate life and possibility anne hathaway',\n",
       " ' shoot at orlando nightclub kill about police say',\n",
       " ' snatch some of these this past week',\n",
       " ' great aicle right on the money',\n",
       " ' baby wa bear welcome to new world cat',\n",
       " ' re',\n",
       " ' even my dear cutys',\n",
       " ' it all about me mentality of our culture',\n",
       " ' wish all of u a very',\n",
       " ' i be thankful for freezer',\n",
       " ' ready for &lta&gt full day of &lthtml&gt from to',\n",
       " ' bihday cake i make for my mom',\n",
       " ' thank you the power of',\n",
       " ' this make me smile cto',\n",
       " ' send good vibe your way happy friday',\n",
       " ' more day till i know im',\n",
       " ' holy city',\n",
       " ' how old be your mom bihday',\n",
       " ' what do you decide',\n",
       " ' get him angry a muslim assassinate not so much',\n",
       " ' we out hereeeee ericprydz',\n",
       " ' thing about we watch the feel etc then back to open for business a usual not re the',\n",
       " ' and you keep tell that only aryan be allow to rape woman youre just a troll',\n",
       " ' speechless when it come to all the shoot recently i guess people have nothing to lose and dont have any respect',\n",
       " ' a forecast the beautiful wave of raalhugandu ha disappear due to construction of the bridge&amp there no hope of it com',\n",
       " ' any minute will condemn the comment of',\n",
       " ' all wonderful a s let today at red rascal',\n",
       " ' vps get multi million dollar bonus while average american lose their home',\n",
       " ' i love and',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' sign up for my first bikram yoga class',\n",
       " ' think think and they will shine from your face like sun beam quote',\n",
       " ' if you want creative worker give them enough time to play',\n",
       " ' my hea prayer be with the orlando victim their family and the lgbt community',\n",
       " ' jan interest in how and ha affect the criminal justice system',\n",
       " ' friend forever egna hem',\n",
       " ' keep up the opposition to',\n",
       " ' love pizza',\n",
       " ' my two love',\n",
       " ' i be abundant',\n",
       " ' fathersday buy thing about h',\n",
       " ' rickityrick rickityrick a the same ole break down wagon be look to find the next mateen to jump on board',\n",
       " ' love this city great day in',\n",
       " ' i guess be talk about progress in the advancement of',\n",
       " ' teen girl kill others injure in downtown oakland shoot be',\n",
       " ' to everyone go through any struggle or pain right now keep your head up and keep on smile youll get through it i hope so',\n",
       " ' good even my darling instagram baby',\n",
       " ' jake be right many ppl lack the will power to take a stand &amp drown the small mind &amp oft evil tendency of past',\n",
       " ' i really hate hipocrit politician which be just about all on all side we deserve better a citizen',\n",
       " ' where be environmental wellbeing the plan be only about site nothng todo with w',\n",
       " ' bihday sway bihday sway bihday bihday',\n",
       " ' now go back to sleep fade away johnmaun listen',\n",
       " ' i wa founate to have this man in my life every single day to all',\n",
       " ' gutted didnt know that you be here tonight i would have be there luck',\n",
       " ' begin to think be a tom brady hater the way he go on about the man',\n",
       " ' the attack in ha cause a lot of pain to many people be not an option we be all very let comfo each other',\n",
       " ' cuteness overload let your life fill with love and sweetness marry life',\n",
       " ' head to',\n",
       " ' be it possible for you to speak in any topic without bring race into it you bigot',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' ours too so much terribly news over the weekend',\n",
       " ' do more of what make you',\n",
       " ' update',\n",
       " ' doe his incredibly obama impression via',\n",
       " ' we the people be fail our fellow citizen',\n",
       " ' staing to pack for my move',\n",
       " ' he be my first love i may not be his but he make me feel like i be',\n",
       " ' just youtu bed them two and they dont get on now',\n",
       " ' if you want to be be kind to the it be a to despise anyone',\n",
       " ' have a nice life im do try to be in it',\n",
       " ' the latest positivementalattitude thank to',\n",
       " ' you want to spark a serious debate about gun violence release the photo of the body inside the club',\n",
       " ' just arrive in blagoevgrad fo conce feel under the',\n",
       " ' a number of our leader be come',\n",
       " ' so to see at yesterday',\n",
       " ' isnt mean to me so that mean he doesnt like me',\n",
       " ' great way to spend a saturday',\n",
       " ' usd could test near term westpac',\n",
       " ' he ha no money campaign ask for k donation by midnight to fight hrcs ad buy',\n",
       " ' today call for champagne',\n",
       " ' rightly so gop hate trump more than obama',\n",
       " ' sunday hug',\n",
       " ' just the of u',\n",
       " ' in that moment she decide she need more a switch to control life response default',\n",
       " ' now play yiruma kiss the rain on music song',\n",
       " ' so simple but so true become all we be capable of be',\n",
       " ' backwho miss me',\n",
       " ' do not i repeat do not put a dog in a hot car and leave them there in the heat keep them home for fuck sake',\n",
       " ' what a splendid me probably not this',\n",
       " ' hour to go until our music video be release',\n",
       " ' he be confirm deadaccording to farrah gray',\n",
       " ' look at journey leave and right foot stage',\n",
       " ' black professor make assumption about an entire race whilst speak for entire race next week the jew',\n",
       " ' back to musicnow play michael jacksonthe lose child unificationsdayand may god all the child &amp their familiessmile',\n",
       " ' buffalo simulation buffalo for you to take in the vicinity of their home to do in this wa',\n",
       " ' when they call u they be really say they be jealous of u w',\n",
       " ' i think you should suspend indefininetly that racist staff',\n",
       " ' *she sit alone and let out a sad sigh*',\n",
       " ' good morning a quick preview of a new that well be launch very soon about this one',\n",
       " ' try on the new today although they feel very similar to v the v be smoother and feel great',\n",
       " ' kiki felt ignore today make kiki dance again',\n",
       " ' the source of my ideal my father day',\n",
       " ' congrats to the thomas on your new home in tehaleh welcome to the lennar family',\n",
       " ' i miss the day when my friend play xbox',\n",
       " ' new case and scraggly hair',\n",
       " ' love our dincwear mum da word for straight up on saturday',\n",
       " ' bihday sexy girl girl on top of boy naked',\n",
       " ' he the only one where the majority of fan like him salty gsw fan just need someone to be about',\n",
       " ' change on a daily basis week until handover network team in soon then av',\n",
       " ' when be this come to singapore',\n",
       " ' sma baby sock gbp get here',\n",
       " ' i like a video from retard want more',\n",
       " ' so woh the wait my wife be',\n",
       " ' lmfao deadass',\n",
       " ' what an amaze interview',\n",
       " ' boasty tuesday last night boom boom and harry hype say get',\n",
       " ' bihday balayya even at that age mass lookcraze in fanson screen romancehe be a legend',\n",
       " ' woohoo',\n",
       " ' outline my theme for my would be poem haha',\n",
       " ' buffalo simulation buffalo for you to take in the vicinity of their home to do in this way you',\n",
       " ' sell one of two paint that be go to get a beautiful new home this week',\n",
       " ' msg of for a business to end',\n",
       " ' why doe slag off child i love your show butreally',\n",
       " ' safe to say im miss sit at home on your own watch listen to the rain be',\n",
       " ' they have be brainwash by the leave who be allow these terrorist to come into america to kill them',\n",
       " ' gd morning',\n",
       " ' friendszxc',\n",
       " ' more day till vacation',\n",
       " ' the secret of a happy marriage be all about find other people le attractive and those who repo',\n",
       " ' python and concurrency be not best friend all the code i have restructure to get concurrency with celery',\n",
       " ' we think so',\n",
       " ' hairy pussy redtube hardcore track',\n",
       " ' let the season binge begin',\n",
       " ' rest in peace',\n",
       " ' a picture be woh a thousand word',\n",
       " ' relax',\n",
       " ' countdown ha begin enteainment',\n",
       " ' beginner super to use',\n",
       " ' dear istore please dont crash down due to heavy overload on',\n",
       " ' never underestimate the power of a good book good wine good food and a great patio',\n",
       " ' istg this be the best cheese ta but dayum expensive',\n",
       " ' my morning office routine will never be the same without my bff',\n",
       " ' i love u take with u all the time in ur',\n",
       " ' weak attempt to cash in on the jaw anniversary',\n",
       " ' non stop election i blame for hide for year pittsburgh california',\n",
       " ' be extremely to reveal the first for pa come this mo',\n",
       " ' minute and count',\n",
       " ' he ha to play ball now the state of our politics it now all a facade a weve lose principality',\n",
       " ' arent protest because a win they do so because trump ha fuhered &amp',\n",
       " ' it a great day to just be nothing but happy',\n",
       " ' have a happy sunday everyone',\n",
       " ' i always help others never myself because other people matter and i clearly dont',\n",
       " ' my bihday weekend even better xx',\n",
       " ' stay strong',\n",
       " ' happy father day',\n",
       " ' aww yeah it all good bing bong bing bong',\n",
       " ' be most for b',\n",
       " ' to the father of my baby i love u happiest of father day via',\n",
       " ' current mood about s web developer meet &amp hire on',\n",
       " ' you know youre get old when you sta to get turn on look at poweools in b&ampq',\n",
       " ' re project busy hand be happy hand',\n",
       " ' bride a woman with a fine prospect of happiness behind her ambrose bierce',\n",
       " ' xoxosamantha via',\n",
       " ' stop rob innocent customer of their hard earn aiime which you guy have no idea on how dey manage to recharge',\n",
       " ' grab your bestie &amp swing by for hour ~ weve miss you',\n",
       " ' be not an racism be people base on race to justify or open against them',\n",
       " ' finally move today',\n",
       " ' and im on my way can we say',\n",
       " ' yay fantastic for',\n",
       " ' lemme find out i have to put on my boiler in the month of june',\n",
       " ' my hea go out to the family of those people kill in the massacre in orlando',\n",
       " ' pre gig drink',\n",
       " ' so we rescue a pup meet rexy',\n",
       " ' these video will get you excite for the future',\n",
       " ' congrats ur wish come true work with superxcited for the song',\n",
       " ' we wish it be',\n",
       " ' i hope you all have a sunnday',\n",
       " ' i once do that gifdidt listen to this song since age',\n",
       " ' usd rise to eye gdt price index and fomc statement',\n",
       " ' how true home food be consider a cheap meal impression be make at restaurant',\n",
       " ' happy bihday to one of the west greatest tupac shakur',\n",
       " ' live your life',\n",
       " ' hr to go to be reveal feel stay tune',\n",
       " ' i have a little taste it wa for sure',\n",
       " ' appletstag bday',\n",
       " ' a tree friend pixiv happy tree',\n",
       " ' happy nd wed anniversary to my amaze husband',\n",
       " ' after a slow sta i have a productive day get back into a routine after a bout of fatigue and flareup be hard',\n",
       " ' i love ramdan and fastingi love the iftars feel and i love the way i feel during ramadan with pray and read quran',\n",
       " ' gbp subdue trade action around',\n",
       " ' with',\n",
       " ' it an unusual cloudy day election day and picnic with friend sunday to everyone',\n",
       " ' gorilla simulator you need to do to adapt to the environment the need to tear the city mat',\n",
       " ' christ be alive in you',\n",
       " ' want leak of udta punjab probe suspect political maker of udt',\n",
       " ' yesterday ozen',\n",
       " ' i can already tell that next week episode be gonna give me intense anxiety',\n",
       " ' youve get nothing to say so you assume that ahahahahahah',\n",
       " ' im just an ordinary girl',\n",
       " ' may light triumph over darkness may this ramadan bring peace harmony and joy',\n",
       " ' any else notice most of the suppoers r pageant girl wyears to internalize a female oppressive system',\n",
       " ' reason that optimist rule the world',\n",
       " ' i hate online application because you dont even get your foot in the door before they tell you no we dont need you',\n",
       " ' i be thankful for flower',\n",
       " ' to top off my crappy day w me not sign paper for my house pop up conce block from work &amp i didnt know',\n",
       " ' beautiful i love live at the beach',\n",
       " ' omg bitch i know you wa go do lil kim actually look like a music video',\n",
       " ' to my',\n",
       " ' man i even have a bunch of coupon too',\n",
       " ' way to sta the day',\n",
       " ' lking forward our water hole watchtower fore',\n",
       " ' the next best seller book',\n",
       " ' this a a fantastic rub for steak',\n",
       " ' touch my tit',\n",
       " ' feel like summer',\n",
       " ' reckon that he go beyond the call of duty with my prezzie',\n",
       " ' dont forget research show that sexist men have psychological problem',\n",
       " ' nice any specific pedagogical change',\n",
       " ' typical city fan',\n",
       " ' no im not attract to asian ppl bc im not asian what if i tell you no im not attract to u bc im not inse race here hoe',\n",
       " ' awlrite mol jealous better than be at cricket pitch all my life',\n",
       " ' prayer go out to eddie and his family',\n",
       " ' my carer be finish in one week *',\n",
       " ' of the',\n",
       " ' wingym',\n",
       " ' glad it all arrive safely',\n",
       " ' who define historically or historic',\n",
       " ' be you really or just pretend to be',\n",
       " ' i finally find a way how to delete old tweet you might find it useful a well',\n",
       " ' what a',\n",
       " ' have you put in your yet if you havent do it and so we can see what make',\n",
       " ' next chapter in life stas soon look at a house today with my fianc',\n",
       " ...]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_lem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(X_train_lem)\n",
    "train_seq = tokenizer.texts_to_sequences(X_train_lem)\n",
    "train_pad = pad_sequences(train_seq, maxlen = max_len)\n",
    "test_seq = tokenizer.texts_to_sequences(X_test_lem)\n",
    "test_pad = pad_sequences(test_seq, maxlen = max_len)\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "embedding_dim = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18246"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_pad[0])\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.Sequential([\n",
    "    keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_len),\n",
    "    keras.layers.Bidirectional(keras.layers.LSTM(100, return_sequences=True)),\n",
    "    keras.layers.Dropout(0.2),\n",
    "    keras.layers.Bidirectional(keras.layers.LSTM(100)),\n",
    "    keras.layers.Dropout(0.2),\n",
    "    keras.layers.Dense(embedding_dim, activation='relu'),\n",
    "    keras.layers.Dense(1, activation='sigmoid')\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "63/63 [==============================] - 104s 1s/step - loss: 0.3407 - accuracy: 0.8963\n",
      "Epoch 2/30\n",
      "63/63 [==============================] - 89s 1s/step - loss: 0.1334 - accuracy: 0.9543\n",
      "Epoch 3/30\n",
      "63/63 [==============================] - 91s 1s/step - loss: 0.0810 - accuracy: 0.9737\n",
      "Epoch 4/30\n",
      "63/63 [==============================] - 119s 2s/step - loss: 0.0552 - accuracy: 0.9829\n",
      "Epoch 5/30\n",
      "63/63 [==============================] - 116s 2s/step - loss: 0.0395 - accuracy: 0.9888\n",
      "Epoch 6/30\n",
      "63/63 [==============================] - 86s 1s/step - loss: 0.0275 - accuracy: 0.9919\n",
      "Epoch 7/30\n",
      "63/63 [==============================] - 89s 1s/step - loss: 0.0213 - accuracy: 0.9941\n",
      "Epoch 8/30\n",
      "63/63 [==============================] - 91s 1s/step - loss: 0.0205 - accuracy: 0.9940\n",
      "Epoch 9/30\n",
      "63/63 [==============================] - 92s 1s/step - loss: 0.0196 - accuracy: 0.9940\n",
      "Epoch 10/30\n",
      "63/63 [==============================] - 91s 1s/step - loss: 0.0204 - accuracy: 0.9940\n",
      "Epoch 11/30\n",
      "63/63 [==============================] - 106s 2s/step - loss: 0.0142 - accuracy: 0.9962\n",
      "Epoch 12/30\n",
      "63/63 [==============================] - 117s 2s/step - loss: 0.0132 - accuracy: 0.9967\n",
      "Epoch 13/30\n",
      "63/63 [==============================] - 100s 2s/step - loss: 0.0123 - accuracy: 0.9968\n",
      "Epoch 14/30\n",
      "63/63 [==============================] - 91s 1s/step - loss: 0.0083 - accuracy: 0.9977\n",
      "Epoch 15/30\n",
      "63/63 [==============================] - 90s 1s/step - loss: 0.0100 - accuracy: 0.9972\n",
      "Epoch 16/30\n",
      "63/63 [==============================] - 91s 1s/step - loss: 0.0065 - accuracy: 0.9980\n",
      "Epoch 17/30\n",
      "63/63 [==============================] - 93s 1s/step - loss: 0.0065 - accuracy: 0.9982\n",
      "Epoch 18/30\n",
      "63/63 [==============================] - 91s 1s/step - loss: 0.0089 - accuracy: 0.9973\n",
      "Epoch 19/30\n",
      "63/63 [==============================] - 91s 1s/step - loss: 0.0089 - accuracy: 0.9972\n",
      "Epoch 20/30\n",
      "63/63 [==============================] - 89s 1s/step - loss: 0.0069 - accuracy: 0.9980\n",
      "Epoch 21/30\n",
      "63/63 [==============================] - 92s 1s/step - loss: 0.0062 - accuracy: 0.9983\n",
      "Epoch 22/30\n",
      "63/63 [==============================] - 87s 1s/step - loss: 0.0052 - accuracy: 0.9982\n",
      "Epoch 23/30\n",
      "63/63 [==============================] - 84s 1s/step - loss: 0.0040 - accuracy: 0.9991\n",
      "Epoch 24/30\n",
      "63/63 [==============================] - 98s 2s/step - loss: 0.0059 - accuracy: 0.9981\n",
      "Epoch 25/30\n",
      "63/63 [==============================] - 79s 1s/step - loss: 0.0056 - accuracy: 0.9983\n",
      "Epoch 26/30\n",
      "63/63 [==============================] - 77s 1s/step - loss: 0.0062 - accuracy: 0.9980\n",
      "Epoch 27/30\n",
      "63/63 [==============================] - 74s 1s/step - loss: 0.0046 - accuracy: 0.9983\n",
      "Epoch 28/30\n",
      "63/63 [==============================] - 88s 1s/step - loss: 0.0047 - accuracy: 0.9986\n",
      "Epoch 29/30\n",
      "63/63 [==============================] - 78s 1s/step - loss: 0.0053 - accuracy: 0.9982\n",
      "Epoch 30/30\n",
      "63/63 [==============================] - 78s 1s/step - loss: 0.0040 - accuracy: 0.9988\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer=\"adam\", metrics=['accuracy'])\n",
    "history = model.fit(train_pad, label, epochs=30, batch_size=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 31, 300)           5473800   \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 31, 200)           320800    \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 31, 200)           0         \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 200)               240800    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 200)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 300)               60300     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 301       \n",
      "=================================================================\n",
      "Total params: 6,096,001\n",
      "Trainable params: 6,096,001\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
      "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0],\n",
       "       [0],\n",
       "       [0],\n",
       "       ...,\n",
       "       [0],\n",
       "       [0],\n",
       "       [0]], dtype=int32)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_keras = model.predict_classes(test_pad)\n",
    "y_pred_keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = pd.DataFrame()\n",
    "targets['id'] =  id\n",
    "targets['label'] = y_pred_keras\n",
    "targets.to_csv('twitter_sentiment_keras.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting autoviml\n",
      "  Downloading autoviml-0.1.682-py3-none-any.whl (125 kB)\n",
      "\u001b[K     |████████████████████████████████| 125 kB 1.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: ipython in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (7.22.0)\n",
      "Collecting textblob\n",
      "  Downloading textblob-0.15.3-py2.py3-none-any.whl (636 kB)\n",
      "\u001b[K     |████████████████████████████████| 636 kB 2.3 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: matplotlib in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (3.3.4)\n",
      "Collecting vaderSentiment\n",
      "  Downloading vaderSentiment-3.3.2-py2.py3-none-any.whl (125 kB)\n",
      "\u001b[K     |████████████████████████████████| 125 kB 2.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: nltk in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (3.6.2)\n",
      "Collecting shap>=0.36.0\n",
      "  Downloading shap-0.39.0.tar.gz (356 kB)\n",
      "\u001b[K     |████████████████████████████████| 356 kB 1.9 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: xgboost>=1.1.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (1.4.2)\n",
      "Requirement already satisfied: scikit-learn>=0.23.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (0.24.2)\n",
      "Requirement already satisfied: seaborn in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (0.11.1)\n",
      "Requirement already satisfied: regex in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (2021.4.4)\n",
      "Requirement already satisfied: pandas in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (1.2.4)\n",
      "Collecting emoji\n",
      "  Downloading emoji-1.2.0-py3-none-any.whl (131 kB)\n",
      "\u001b[K     |████████████████████████████████| 131 kB 502 kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: beautifulsoup4 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (4.9.3)\n",
      "Requirement already satisfied: xlrd in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (2.0.1)\n",
      "Collecting imbalanced-learn>=0.7\n",
      "  Downloading imbalanced_learn-0.8.0-py3-none-any.whl (206 kB)\n",
      "\u001b[K     |████████████████████████████████| 206 kB 1.7 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: jupyter in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (1.0.0)\n",
      "Requirement already satisfied: catboost in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autoviml) (0.26)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from imbalanced-learn>=0.7->autoviml) (1.19.5)\n",
      "Requirement already satisfied: joblib>=0.11 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from imbalanced-learn>=0.7->autoviml) (1.0.1)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from imbalanced-learn>=0.7->autoviml) (1.6.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from scikit-learn>=0.23.1->autoviml) (2.1.0)\n",
      "Requirement already satisfied: tqdm>4.25.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from shap>=0.36.0->autoviml) (4.59.0)\n",
      "Collecting slicer==0.0.7\n",
      "  Downloading slicer-0.0.7-py3-none-any.whl (14 kB)\n",
      "Requirement already satisfied: numba in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from shap>=0.36.0->autoviml) (0.53.1)\n",
      "Requirement already satisfied: cloudpickle in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from shap>=0.36.0->autoviml) (1.6.0)\n",
      "Requirement already satisfied: soupsieve>1.2 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from beautifulsoup4->autoviml) (2.2.1)\n",
      "Requirement already satisfied: graphviz in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from catboost->autoviml) (0.16)\n",
      "Requirement already satisfied: plotly in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from catboost->autoviml) (4.14.3)\n",
      "Requirement already satisfied: six in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from catboost->autoviml) (1.15.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from pandas->autoviml) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from pandas->autoviml) (2021.1)\n",
      "Requirement already satisfied: pexpect>4.3 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (4.8.0)\n",
      "Requirement already satisfied: decorator in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (5.0.9)\n",
      "Requirement already satisfied: jedi>=0.16 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (0.17.2)\n",
      "Requirement already satisfied: appnope in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (0.1.2)\n",
      "Requirement already satisfied: pickleshare in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (0.7.5)\n",
      "Requirement already satisfied: traitlets>=4.2 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (5.0.5)\n",
      "Requirement already satisfied: pygments in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (2.9.0)\n",
      "Requirement already satisfied: setuptools>=18.5 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (52.0.0.post20210125)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (3.0.17)\n",
      "Requirement already satisfied: backcall in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipython->autoviml) (0.2.0)\n",
      "Requirement already satisfied: parso<0.8.0,>=0.7.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jedi>=0.16->ipython->autoviml) (0.7.0)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from pexpect>4.3->ipython->autoviml) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->autoviml) (0.2.5)\n",
      "Requirement already satisfied: ipython-genutils in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from traitlets>=4.2->ipython->autoviml) (0.2.0)\n",
      "Requirement already satisfied: ipykernel in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jupyter->autoviml) (5.3.4)\n",
      "Requirement already satisfied: nbconvert in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jupyter->autoviml) (6.0.7)\n",
      "Requirement already satisfied: qtconsole in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jupyter->autoviml) (5.0.3)\n",
      "Requirement already satisfied: jupyter-console in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jupyter->autoviml) (6.4.0)\n",
      "Requirement already satisfied: ipywidgets in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jupyter->autoviml) (7.6.3)\n",
      "Requirement already satisfied: notebook in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jupyter->autoviml) (6.4.0)\n",
      "Requirement already satisfied: tornado>=4.2 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipykernel->jupyter->autoviml) (6.1)\n",
      "Requirement already satisfied: jupyter-client in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipykernel->jupyter->autoviml) (6.1.12)\n",
      "Requirement already satisfied: widgetsnbextension~=3.5.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipywidgets->jupyter->autoviml) (3.5.1)\n",
      "Requirement already satisfied: nbformat>=4.2.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipywidgets->jupyter->autoviml) (5.1.3)\n",
      "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from ipywidgets->jupyter->autoviml) (1.0.0)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbformat>=4.2.0->ipywidgets->jupyter->autoviml) (3.2.0)\n",
      "Requirement already satisfied: jupyter-core in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbformat>=4.2.0->ipywidgets->jupyter->autoviml) (4.7.1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: attrs>=17.4.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter->autoviml) (21.2.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets->jupyter->autoviml) (0.17.3)\n",
      "Requirement already satisfied: argon2-cffi in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from notebook->jupyter->autoviml) (20.1.0)\n",
      "Requirement already satisfied: jinja2 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from notebook->jupyter->autoviml) (3.0.0)\n",
      "Requirement already satisfied: Send2Trash>=1.5.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from notebook->jupyter->autoviml) (1.5.0)\n",
      "Requirement already satisfied: prometheus-client in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from notebook->jupyter->autoviml) (0.10.1)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from notebook->jupyter->autoviml) (0.9.4)\n",
      "Requirement already satisfied: pyzmq>=17 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from notebook->jupyter->autoviml) (20.0.0)\n",
      "Requirement already satisfied: cffi>=1.0.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from argon2-cffi->notebook->jupyter->autoviml) (1.14.5)\n",
      "Requirement already satisfied: pycparser in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from cffi>=1.0.0->argon2-cffi->notebook->jupyter->autoviml) (2.20)\n",
      "Requirement already satisfied: MarkupSafe>=2.0.0rc2 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from jinja2->notebook->jupyter->autoviml) (2.0.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from matplotlib->autoviml) (2.4.7)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from matplotlib->autoviml) (8.2.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from matplotlib->autoviml) (1.3.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from matplotlib->autoviml) (0.10.0)\n",
      "Requirement already satisfied: testpath in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->jupyter->autoviml) (0.4.4)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->jupyter->autoviml) (0.8.4)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->jupyter->autoviml) (1.4.3)\n",
      "Requirement already satisfied: defusedxml in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->jupyter->autoviml) (0.7.1)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->jupyter->autoviml) (0.3)\n",
      "Requirement already satisfied: bleach in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->jupyter->autoviml) (3.3.0)\n",
      "Requirement already satisfied: jupyterlab-pygments in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->jupyter->autoviml) (0.1.2)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->jupyter->autoviml) (0.5.3)\n",
      "Requirement already satisfied: nest-asyncio in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert->jupyter->autoviml) (1.5.1)\n",
      "Requirement already satisfied: async-generator in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert->jupyter->autoviml) (1.10)\n",
      "Requirement already satisfied: webencodings in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from bleach->nbconvert->jupyter->autoviml) (0.5.1)\n",
      "Requirement already satisfied: packaging in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from bleach->nbconvert->jupyter->autoviml) (20.9)\n",
      "Requirement already satisfied: click in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from nltk->autoviml) (8.0.1)\n",
      "Requirement already satisfied: llvmlite<0.37,>=0.36.0rc1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from numba->shap>=0.36.0->autoviml) (0.36.0)\n",
      "Requirement already satisfied: retrying>=1.3.3 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from plotly->catboost->autoviml) (1.3.3)\n",
      "Requirement already satisfied: qtpy in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from qtconsole->jupyter->autoviml) (1.9.0)\n",
      "Requirement already satisfied: requests in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from vaderSentiment->autoviml) (2.25.1)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from requests->vaderSentiment->autoviml) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from requests->vaderSentiment->autoviml) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from requests->vaderSentiment->autoviml) (2021.5.30)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from requests->vaderSentiment->autoviml) (1.26.4)\n",
      "Building wheels for collected packages: shap\n",
      "  Building wheel for shap (setup.py) ... \u001b[?25lerror\n",
      "\u001b[31m  ERROR: Command errored out with exit status 1:\n",
      "   command: /Users/lakshmipriya/opt/anaconda3/bin/python -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-install-8677z7ot/shap_35e9607eb22b4a49ac42ee01e6bb293a/setup.py'\"'\"'; __file__='\"'\"'/private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-install-8677z7ot/shap_35e9607eb22b4a49ac42ee01e6bb293a/setup.py'\"'\"';f = getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__) if os.path.exists(__file__) else io.StringIO('\"'\"'from setuptools import setup; setup()'\"'\"');code = f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' bdist_wheel -d /private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-wheel-xaxj3tlg\n",
      "       cwd: /private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-install-8677z7ot/shap_35e9607eb22b4a49ac42ee01e6bb293a/\n",
      "  Complete output (121 lines):\n",
      "  The nvcc binary could not be located in your $PATH. Either  add it to your path, or set $CUDAHOME to enable CUDA\n",
      "  Error building cuda module: TypeError('cannot unpack non-iterable NoneType object')\n",
      "  WARNING: Could not compile cuda extensions\n",
      "  running bdist_wheel\n",
      "  running build\n",
      "  running build_py\n",
      "  creating build\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "  copying shap/datasets.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "  copying shap/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "  copying shap/links.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "  copying shap/_explanation.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "  copying shap/_serializable.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_explainer.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_partition.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_gradient.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/mimic.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_sampling.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_linear.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_permutation.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_gpu_tree.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_kernel.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_additive.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/tf_utils.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_exact.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/_tree.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  copying shap/explainers/pytree.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "  copying shap/explainers/other/_coefficent.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "  copying shap/explainers/other/_maple.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "  copying shap/explainers/other/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "  copying shap/explainers/other/_random.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "  copying shap/explainers/other/_treegain.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "  copying shap/explainers/other/_lime.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/explainers/_deep\n",
      "  copying shap/explainers/_deep/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/_deep\n",
      "  copying shap/explainers/_deep/deep_pytorch.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/_deep\n",
      "  copying shap/explainers/_deep/deep_tf.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/_deep\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_heatmap.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_partial_dependence.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_monitoring.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_violin.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_decision.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_text.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_force_matplotlib.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_force.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_group_difference.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_embedding.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_beeswarm.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_bar.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_image.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_scatter.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_waterfall.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_labels.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  copying shap/plots/_utils.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/plots/colors\n",
      "  copying shap/plots/colors/_colors.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/colors\n",
      "  copying shap/plots/colors/_colorconv.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/colors\n",
      "  copying shap/plots/colors/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/colors\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  copying shap/benchmark/metrics.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  copying shap/benchmark/models.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  copying shap/benchmark/methods.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  copying shap/benchmark/framework.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  copying shap/benchmark/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  copying shap/benchmark/measures.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  copying shap/benchmark/plots.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  copying shap/benchmark/experiments.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  copying shap/benchmark/perturbation.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  copying shap/maskers/_fixed_composite.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  copying shap/maskers/_fixed.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  copying shap/maskers/_masker.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  copying shap/maskers/_text.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  copying shap/maskers/_tabular.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  copying shap/maskers/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  copying shap/maskers/_image.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  copying shap/maskers/_output_composite.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  copying shap/maskers/_composite.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  copying shap/utils/transformers.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  copying shap/utils/_clustering.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  copying shap/utils/_keras.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  copying shap/utils/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  copying shap/utils/_show_progress.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  copying shap/utils/_masked_model.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  copying shap/utils/_legacy.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  copying shap/utils/_general.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  copying shap/utils/image.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/actions\n",
      "  copying shap/actions/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/actions\n",
      "  copying shap/actions/_action.py -> build/lib.macosx-10.9-x86_64-3.8/shap/actions\n",
      "  copying shap/actions/_optimizer.py -> build/lib.macosx-10.9-x86_64-3.8/shap/actions\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "  copying shap/models/_topk_lm.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "  copying shap/models/_transformers_pipeline.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "  copying shap/models/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "  copying shap/models/_text_generation.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "  copying shap/models/_model.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "  copying shap/models/_teacher_forcing.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/plots/resources\n",
      "  copying shap/plots/resources/bundle.js -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/resources\n",
      "  copying shap/plots/resources/logoSmallGray.png -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/resources\n",
      "  creating build/lib.macosx-10.9-x86_64-3.8/shap/cext\n",
      "  copying shap/cext/tree_shap.h -> build/lib.macosx-10.9-x86_64-3.8/shap/cext\n",
      "  running build_ext\n",
      "  numpy.get_include() /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages/numpy/core/include\n",
      "  building 'shap._cext' extension\n",
      "  creating build/temp.macosx-10.9-x86_64-3.8\n",
      "  creating build/temp.macosx-10.9-x86_64-3.8/shap\n",
      "  creating build/temp.macosx-10.9-x86_64-3.8/shap/cext\n",
      "  gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/Users/lakshmipriya/opt/anaconda3/include -arch x86_64 -I/Users/lakshmipriya/opt/anaconda3/include -arch x86_64 -I/Users/lakshmipriya/opt/anaconda3/include/python3.8 -I/Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages/numpy/core/include -c shap/cext/_cext.cc -o build/temp.macosx-10.9-x86_64-3.8/shap/cext/_cext.o\n",
      "  xcrun: error: active developer path (\"/Applications/Xcode.app/Contents/Developer\") does not exist\n",
      "  Use `sudo xcode-select --switch path/to/Xcode.app` to specify the Xcode that you wish to use for command line developer tools, or use `xcode-select --install` to install the standalone command line developer tools.\n",
      "  See `man xcode-select` for more details.\n",
      "  error: command 'gcc' failed with exit status 1\n",
      "  ----------------------------------------\u001b[0m\n",
      "\u001b[31m  ERROR: Failed building wheel for shap\u001b[0m\n",
      "\u001b[?25h  Running setup.py clean for shap\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to build shap\n",
      "Installing collected packages: slicer, vaderSentiment, textblob, shap, imbalanced-learn, emoji, autoviml\n",
      "    Running setup.py install for shap ... \u001b[?25lerror\n",
      "\u001b[31m    ERROR: Command errored out with exit status 1:\n",
      "     command: /Users/lakshmipriya/opt/anaconda3/bin/python -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-install-8677z7ot/shap_35e9607eb22b4a49ac42ee01e6bb293a/setup.py'\"'\"'; __file__='\"'\"'/private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-install-8677z7ot/shap_35e9607eb22b4a49ac42ee01e6bb293a/setup.py'\"'\"';f = getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__) if os.path.exists(__file__) else io.StringIO('\"'\"'from setuptools import setup; setup()'\"'\"');code = f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' install --record /private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-record-94ghvdbo/install-record.txt --single-version-externally-managed --compile --install-headers /Users/lakshmipriya/opt/anaconda3/include/python3.8/shap\n",
      "         cwd: /private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-install-8677z7ot/shap_35e9607eb22b4a49ac42ee01e6bb293a/\n",
      "    Complete output (121 lines):\n",
      "    The nvcc binary could not be located in your $PATH. Either  add it to your path, or set $CUDAHOME to enable CUDA\n",
      "    Error building cuda module: TypeError('cannot unpack non-iterable NoneType object')\n",
      "    WARNING: Could not compile cuda extensions\n",
      "    running install\n",
      "    running build\n",
      "    running build_py\n",
      "    creating build\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "    copying shap/datasets.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "    copying shap/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "    copying shap/links.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "    copying shap/_explanation.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "    copying shap/_serializable.py -> build/lib.macosx-10.9-x86_64-3.8/shap\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_explainer.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_partition.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_gradient.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/mimic.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_sampling.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_linear.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_permutation.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_gpu_tree.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_kernel.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_additive.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/tf_utils.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_exact.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/_tree.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    copying shap/explainers/pytree.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "    copying shap/explainers/other/_coefficent.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "    copying shap/explainers/other/_maple.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "    copying shap/explainers/other/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "    copying shap/explainers/other/_random.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "    copying shap/explainers/other/_treegain.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "    copying shap/explainers/other/_lime.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/other\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/explainers/_deep\n",
      "    copying shap/explainers/_deep/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/_deep\n",
      "    copying shap/explainers/_deep/deep_pytorch.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/_deep\n",
      "    copying shap/explainers/_deep/deep_tf.py -> build/lib.macosx-10.9-x86_64-3.8/shap/explainers/_deep\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_heatmap.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_partial_dependence.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_monitoring.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_violin.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_decision.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_text.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_force_matplotlib.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_force.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_group_difference.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_embedding.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_beeswarm.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_bar.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_image.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_scatter.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_waterfall.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_labels.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    copying shap/plots/_utils.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/plots/colors\n",
      "    copying shap/plots/colors/_colors.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/colors\n",
      "    copying shap/plots/colors/_colorconv.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/colors\n",
      "    copying shap/plots/colors/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/colors\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    copying shap/benchmark/metrics.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    copying shap/benchmark/models.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    copying shap/benchmark/methods.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    copying shap/benchmark/framework.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    copying shap/benchmark/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    copying shap/benchmark/measures.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    copying shap/benchmark/plots.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    copying shap/benchmark/experiments.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    copying shap/benchmark/perturbation.py -> build/lib.macosx-10.9-x86_64-3.8/shap/benchmark\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    copying shap/maskers/_fixed_composite.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    copying shap/maskers/_fixed.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    copying shap/maskers/_masker.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    copying shap/maskers/_text.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    copying shap/maskers/_tabular.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    copying shap/maskers/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    copying shap/maskers/_image.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    copying shap/maskers/_output_composite.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    copying shap/maskers/_composite.py -> build/lib.macosx-10.9-x86_64-3.8/shap/maskers\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    copying shap/utils/transformers.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    copying shap/utils/_clustering.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    copying shap/utils/_keras.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    copying shap/utils/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    copying shap/utils/_show_progress.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    copying shap/utils/_masked_model.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    copying shap/utils/_legacy.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    copying shap/utils/_general.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    copying shap/utils/image.py -> build/lib.macosx-10.9-x86_64-3.8/shap/utils\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/actions\n",
      "    copying shap/actions/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/actions\n",
      "    copying shap/actions/_action.py -> build/lib.macosx-10.9-x86_64-3.8/shap/actions\n",
      "    copying shap/actions/_optimizer.py -> build/lib.macosx-10.9-x86_64-3.8/shap/actions\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "    copying shap/models/_topk_lm.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "    copying shap/models/_transformers_pipeline.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "    copying shap/models/__init__.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "    copying shap/models/_text_generation.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "    copying shap/models/_model.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "    copying shap/models/_teacher_forcing.py -> build/lib.macosx-10.9-x86_64-3.8/shap/models\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/plots/resources\n",
      "    copying shap/plots/resources/bundle.js -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/resources\n",
      "    copying shap/plots/resources/logoSmallGray.png -> build/lib.macosx-10.9-x86_64-3.8/shap/plots/resources\n",
      "    creating build/lib.macosx-10.9-x86_64-3.8/shap/cext\n",
      "    copying shap/cext/tree_shap.h -> build/lib.macosx-10.9-x86_64-3.8/shap/cext\n",
      "    running build_ext\n",
      "    numpy.get_include() /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages/numpy/core/include\n",
      "    building 'shap._cext' extension\n",
      "    creating build/temp.macosx-10.9-x86_64-3.8\n",
      "    creating build/temp.macosx-10.9-x86_64-3.8/shap\n",
      "    creating build/temp.macosx-10.9-x86_64-3.8/shap/cext\n",
      "    gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/Users/lakshmipriya/opt/anaconda3/include -arch x86_64 -I/Users/lakshmipriya/opt/anaconda3/include -arch x86_64 -I/Users/lakshmipriya/opt/anaconda3/include/python3.8 -I/Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages/numpy/core/include -c shap/cext/_cext.cc -o build/temp.macosx-10.9-x86_64-3.8/shap/cext/_cext.o\n",
      "    xcrun: error: active developer path (\"/Applications/Xcode.app/Contents/Developer\") does not exist\n",
      "    Use `sudo xcode-select --switch path/to/Xcode.app` to specify the Xcode that you wish to use for command line developer tools, or use `xcode-select --install` to install the standalone command line developer tools.\n",
      "    See `man xcode-select` for more details.\n",
      "    error: command 'gcc' failed with exit status 1\n",
      "    ----------------------------------------\u001b[0m\n",
      "\u001b[31mERROR: Command errored out with exit status 1: /Users/lakshmipriya/opt/anaconda3/bin/python -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-install-8677z7ot/shap_35e9607eb22b4a49ac42ee01e6bb293a/setup.py'\"'\"'; __file__='\"'\"'/private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-install-8677z7ot/shap_35e9607eb22b4a49ac42ee01e6bb293a/setup.py'\"'\"';f = getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__) if os.path.exists(__file__) else io.StringIO('\"'\"'from setuptools import setup; setup()'\"'\"');code = f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' install --record /private/var/folders/2_/9q01y5c15hg79wb9ypvwk5280000gn/T/pip-record-94ghvdbo/install-record.txt --single-version-externally-managed --compile --install-headers /Users/lakshmipriya/opt/anaconda3/include/python3.8/shap Check the logs for full command output.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[?25h"
     ]
    }
   ],
   "source": [
    "!pip install autoviml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(<pip._vendor.urllib3.connection.HTTPSConnection object at 0x108601c10>, 'Connection to pypi.org timed out. (connect timeout=15)')': /simple/autonlp/\u001b[0m\n",
      "\u001b[33mWARNING: Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(<pip._vendor.urllib3.connection.HTTPSConnection object at 0x108601a90>, 'Connection to pypi.org timed out. (connect timeout=15)')': /simple/autonlp/\u001b[0m\n",
      "\u001b[33mWARNING: Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ConnectTimeoutError(<pip._vendor.urllib3.connection.HTTPSConnection object at 0x108601b20>, 'Connection to pypi.org timed out. (connect timeout=15)')': /simple/autonlp/\u001b[0m\n",
      "Collecting autonlp\n",
      "  Downloading autonlp-0.2.7-py3-none-any.whl (33 kB)\n",
      "Collecting prettytable==2.0.0\n",
      "  Downloading prettytable-2.0.0-py3-none-any.whl (22 kB)\n",
      "Requirement already satisfied: requests==2.25.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from autonlp) (2.25.1)\n",
      "Collecting tqdm==4.56.0\n",
      "  Downloading tqdm-4.56.0-py2.py3-none-any.whl (72 kB)\n",
      "\u001b[K     |████████████████████████████████| 72 kB 99 kB/s eta 0:00:01\n",
      "\u001b[?25hCollecting loguru==0.5.3\n",
      "  Downloading loguru-0.5.3-py3-none-any.whl (57 kB)\n",
      "\u001b[K     |████████████████████████████████| 57 kB 335 kB/s eta 0:00:01\n",
      "\u001b[?25hCollecting huggingface-hub<0.1.0\n",
      "  Downloading huggingface_hub-0.0.10-py3-none-any.whl (37 kB)\n",
      "Requirement already satisfied: setuptools in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from prettytable==2.0.0->autonlp) (52.0.0.post20210125)\n",
      "Requirement already satisfied: wcwidth in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from prettytable==2.0.0->autonlp) (0.2.5)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from requests==2.25.1->autonlp) (4.0.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from requests==2.25.1->autonlp) (1.26.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from requests==2.25.1->autonlp) (2021.5.30)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from requests==2.25.1->autonlp) (2.10)\n",
      "Requirement already satisfied: filelock in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from huggingface-hub<0.1.0->autonlp) (3.0.12)\n",
      "Requirement already satisfied: typing-extensions in /Users/lakshmipriya/opt/anaconda3/lib/python3.8/site-packages (from huggingface-hub<0.1.0->autonlp) (3.7.4.3)\n",
      "Installing collected packages: tqdm, prettytable, loguru, huggingface-hub, autonlp\n",
      "  Attempting uninstall: tqdm\n",
      "    Found existing installation: tqdm 4.59.0\n",
      "    Uninstalling tqdm-4.59.0:\n",
      "      Successfully uninstalled tqdm-4.59.0\n",
      "Successfully installed autonlp-0.2.7 huggingface-hub-0.0.10 loguru-0.5.3 prettytable-2.0.0 tqdm-4.56.0\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install -U autonlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import autonlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train3, test3 = train_test_split(df, test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}