News Classification using NLP part 3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Train Test Validation Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import nltk\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "import distance\n",
    "from nltk.stem import WordNetLemmatizer    # lemmatize\n",
    "from wordcloud import WordCloud, STOPWORDS\n",
    "import string \n",
    "from sklearn.feature_extraction.text import CountVectorizer #Bag of words\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer # Tfid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Class Index</th>\n",
       "      <th>Summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>['wall', 'st', 'bear', 'claw', 'back', 'black'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>['carlyle', 'look', 'toward', 'commercial', 'a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>['oil', 'economy', 'cloud', 'stock', 'outlook'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>['iraq', 'halt', 'oil', 'export', 'main', 'sou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>['oil', 'price', 'soar', 'time', 'record', 'po...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Class Index                                            Summary\n",
       "0           0            3  ['wall', 'st', 'bear', 'claw', 'back', 'black'...\n",
       "1           1            3  ['carlyle', 'look', 'toward', 'commercial', 'a...\n",
       "2           2            3  ['oil', 'economy', 'cloud', 'stock', 'outlook'...\n",
       "3           3            3  ['iraq', 'halt', 'oil', 'export', 'main', 'sou...\n",
       "4           4            3  ['oil', 'price', 'soar', 'time', 'record', 'po..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('df.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(['Unnamed: 0'],axis = 'columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(127600, 2)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class Index</th>\n",
       "      <th>Summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>['wall', 'st', 'bear', 'claw', 'back', 'black'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>['carlyle', 'look', 'toward', 'commercial', 'a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Class Index                                            Summary\n",
       "0            3  ['wall', 'st', 'bear', 'claw', 'back', 'black'...\n",
       "1            3  ['carlyle', 'look', 'toward', 'commercial', 'a..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test  = train_test_split(df,test_size=0.2,stratify = df['Class Index'])\n",
    "X_train, X_val = train_test_split(X_train,test_size=0.20,stratify = X_train['Class Index'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What does Stratify = True in train test split does ?\n",
    "\n",
    "So , it bascily does sampling in proper manner so that we get same amount of values in each sets , we can check it by [y_train.value_counts()] to check our output labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(81664, 2)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25520, 2)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20416, 2)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    20416\n",
       "3    20416\n",
       "2    20416\n",
       "1    20416\n",
       "Name: Class Index, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train['Class Index'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    6380\n",
       "3    6380\n",
       "2    6380\n",
       "1    6380\n",
       "Name: Class Index, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test['Class Index'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    5104\n",
       "3    5104\n",
       "2    5104\n",
       "1    5104\n",
       "Name: Class Index, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val['Class Index'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6 Creating the Models\n",
    "1: Tf-idf\n",
    "\n",
    "2: Uni-gram,bi-gram,n-gram\n",
    "\n",
    "3: Selecting max_features out of the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6.1 TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Here i have created a small Blog on TF-IDF :\n",
    "https://niharjamdar.medium.com/tf-idf-term-frequency-and-inverse-document-frequency-56a0289d2fb6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"tfidf.png\" width=\"700\" height=\"200\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer_tf = TfidfVectorizer(tokenizer=None,stop_words=None,max_df=0.75,max_features=2000,lowercase=False,ngram_range=(1,2))\n",
    "train_vectors = vectorizer_tf.fit_transform(X_train.Summary)   # For train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_vectors = vectorizer_tf.transform(X_test.Summary)   # for test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_vectors = vectorizer_tf.transform(X_val.Summary)    # for validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X_train: (81664, 2)\n",
      "Shape of X_test: (25520, 2)\n",
      "Shape of X_val: (20416, 2)\n",
      "Shape of train_vectors: (81664, 2000)\n",
      "Shape of test_vectors: (25520, 2000)\n",
      "Shape of val_vectors: (20416, 2000)\n"
     ]
    }
   ],
   "source": [
    "print('Shape of X_train:',X_train.shape)\n",
    "print('Shape of X_test:',X_test.shape)\n",
    "print('Shape of X_val:',X_val.shape)\n",
    "print('Shape of train_vectors:',train_vectors.shape)\n",
    "print('Shape of test_vectors:',test_vectors.shape)\n",
    "print('Shape of val_vectors:',val_vectors.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### What is max_df = .75\n",
    "max_df is used for removing data values that appear too frequently, also known as \"corpus-specific stop words\". For example: max_df = 0.50 means \"It ignores terms that appear in more than 50% of the documents\". max_df = 25 means \"It ignores terms that appear in more than 25 documents\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### One small example of n-gram"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "v = CountVectorizer(ngram_range=(1, 2))\n",
    " pprint(v.fit([\"an apple a day keeps the doctor away\"]).vocabulary_)\n",
    "{u'an': 0,\n",
    " u'an apple': 1,\n",
    " u'apple': 2,\n",
    " u'apple day': 3,\n",
    " u'away': 4,\n",
    " u'day': 5,\n",
    " u'day keeps': 6,\n",
    " u'doctor': 7,\n",
    " u'doctor away': 8,\n",
    " u'keeps': 9,\n",
    " u'keeps the': 10,\n",
    " u'the': 11,\n",
    " u'the doctor': 12}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['able',\n",
       " 'abu',\n",
       " 'abuse',\n",
       " 'access',\n",
       " 'according',\n",
       " 'account',\n",
       " 'accounting',\n",
       " 'accused',\n",
       " 'acquire',\n",
       " 'acquisition',\n",
       " 'across',\n",
       " 'act',\n",
       " 'action',\n",
       " 'activity',\n",
       " 'ad',\n",
       " 'add',\n",
       " 'added',\n",
       " 'administration',\n",
       " 'advance',\n",
       " 'advanced',\n",
       " 'advertising',\n",
       " 'afghan',\n",
       " 'afghanistan',\n",
       " 'afp',\n",
       " 'afp afp',\n",
       " 'africa',\n",
       " 'african',\n",
       " 'afternoon',\n",
       " 'age',\n",
       " 'agency',\n",
       " 'agent',\n",
       " 'ago',\n",
       " 'agree',\n",
       " 'agreed',\n",
       " 'agreement',\n",
       " 'agrees',\n",
       " 'ahead',\n",
       " 'aid',\n",
       " 'aim',\n",
       " 'aimed',\n",
       " 'air',\n",
       " 'airbus',\n",
       " 'aircraft',\n",
       " 'airline',\n",
       " 'airport',\n",
       " 'airway',\n",
       " 'al',\n",
       " 'al qaeda',\n",
       " 'allegation',\n",
       " 'alleged',\n",
       " 'alliance',\n",
       " 'allow',\n",
       " 'allowed',\n",
       " 'allows',\n",
       " 'ally',\n",
       " 'almost',\n",
       " 'along',\n",
       " 'already',\n",
       " 'also',\n",
       " 'although',\n",
       " 'amazon',\n",
       " 'amd',\n",
       " 'america',\n",
       " 'american',\n",
       " 'amid',\n",
       " 'among',\n",
       " 'amp',\n",
       " 'anaheim',\n",
       " 'analyst',\n",
       " 'angel',\n",
       " 'angeles',\n",
       " 'annan',\n",
       " 'announce',\n",
       " 'announced',\n",
       " 'announcement',\n",
       " 'announces',\n",
       " 'annual',\n",
       " 'another',\n",
       " 'anti',\n",
       " 'antitrust',\n",
       " 'aol',\n",
       " 'ap',\n",
       " 'ap ap',\n",
       " 'apparently',\n",
       " 'appeal',\n",
       " 'appeared',\n",
       " 'appears',\n",
       " 'apple',\n",
       " 'apple computer',\n",
       " 'application',\n",
       " 'approach',\n",
       " 'approval',\n",
       " 'approved',\n",
       " 'arab',\n",
       " 'arafat',\n",
       " 'area',\n",
       " 'argentina',\n",
       " 'arial',\n",
       " 'arial helvetica',\n",
       " 'ariel',\n",
       " 'ariel sharon',\n",
       " 'arizona',\n",
       " 'arm',\n",
       " 'armed',\n",
       " 'army',\n",
       " 'around',\n",
       " 'arrest',\n",
       " 'arrested',\n",
       " 'arrived',\n",
       " 'arsenal',\n",
       " 'asia',\n",
       " 'asian',\n",
       " 'ask',\n",
       " 'asked',\n",
       " 'assault',\n",
       " 'asset',\n",
       " 'associate',\n",
       " 'associated',\n",
       " 'association',\n",
       " 'astronaut',\n",
       " 'astros',\n",
       " 'athens',\n",
       " 'atlanta',\n",
       " 'atlantic',\n",
       " 'atomic',\n",
       " 'attack',\n",
       " 'attempt',\n",
       " 'attorney',\n",
       " 'attorney general',\n",
       " 'auction',\n",
       " 'aug',\n",
       " 'august',\n",
       " 'australia',\n",
       " 'australian',\n",
       " 'authority',\n",
       " 'auto',\n",
       " 'available',\n",
       " 'average',\n",
       " 'avoid',\n",
       " 'awaited',\n",
       " 'award',\n",
       " 'away',\n",
       " 'back',\n",
       " 'backed',\n",
       " 'bad',\n",
       " 'baghdad',\n",
       " 'ball',\n",
       " 'ballot',\n",
       " 'baltimore',\n",
       " 'ban',\n",
       " 'bangladesh',\n",
       " 'bank',\n",
       " 'banking',\n",
       " 'bankruptcy',\n",
       " 'bar',\n",
       " 'barrel',\n",
       " 'base',\n",
       " 'baseball',\n",
       " 'based',\n",
       " 'basketball',\n",
       " 'battle',\n",
       " 'bay',\n",
       " 'bear',\n",
       " 'beat',\n",
       " 'beating',\n",
       " 'became',\n",
       " 'become',\n",
       " 'becoming',\n",
       " 'began',\n",
       " 'begin',\n",
       " 'beginning',\n",
       " 'behind',\n",
       " 'beijing',\n",
       " 'belief',\n",
       " 'believe',\n",
       " 'bell',\n",
       " 'benefit',\n",
       " 'best',\n",
       " 'better',\n",
       " 'bid',\n",
       " 'big',\n",
       " 'biggest',\n",
       " 'bill',\n",
       " 'billion',\n",
       " 'billion dollar',\n",
       " 'bird',\n",
       " 'bit',\n",
       " 'black',\n",
       " 'blair',\n",
       " 'blast',\n",
       " 'block',\n",
       " 'blow',\n",
       " 'blue',\n",
       " 'bn',\n",
       " 'board',\n",
       " 'body',\n",
       " 'boeing',\n",
       " 'bomb',\n",
       " 'bomber',\n",
       " 'bombing',\n",
       " 'bond',\n",
       " 'book',\n",
       " 'boost',\n",
       " 'border',\n",
       " 'bos',\n",
       " 'boston',\n",
       " 'boston red',\n",
       " 'bought',\n",
       " 'bowl',\n",
       " 'box',\n",
       " 'boy',\n",
       " 'brand',\n",
       " 'brave',\n",
       " 'brazil',\n",
       " 'break',\n",
       " 'breaking',\n",
       " 'bring',\n",
       " 'britain',\n",
       " 'british',\n",
       " 'broadband',\n",
       " 'broke',\n",
       " 'broker',\n",
       " 'brother',\n",
       " 'brought',\n",
       " 'brown',\n",
       " 'browser',\n",
       " 'bryant',\n",
       " 'budget',\n",
       " 'build',\n",
       " 'building',\n",
       " 'built',\n",
       " 'bus',\n",
       " 'bush',\n",
       " 'bush administration',\n",
       " 'business',\n",
       " 'buy',\n",
       " 'buying',\n",
       " 'ca',\n",
       " 'cabinet',\n",
       " 'cable',\n",
       " 'calif',\n",
       " 'california',\n",
       " 'call',\n",
       " 'called',\n",
       " 'calling',\n",
       " 'came',\n",
       " 'camera',\n",
       " 'camp',\n",
       " 'campaign',\n",
       " 'canada',\n",
       " 'canadian',\n",
       " 'canadian press',\n",
       " 'cancer',\n",
       " 'candidate',\n",
       " 'capacity',\n",
       " 'capital',\n",
       " 'captain',\n",
       " 'car',\n",
       " 'car bomb',\n",
       " 'card',\n",
       " 'cardinal',\n",
       " 'care',\n",
       " 'career',\n",
       " 'carolina',\n",
       " 'carrier',\n",
       " 'carry',\n",
       " 'case',\n",
       " 'cash',\n",
       " 'cause',\n",
       " 'caused',\n",
       " 'cbs',\n",
       " 'cbs mw',\n",
       " 'cell',\n",
       " 'cell phone',\n",
       " 'celtic',\n",
       " 'cent',\n",
       " 'center',\n",
       " 'central',\n",
       " 'century',\n",
       " 'ceo',\n",
       " 'chain',\n",
       " 'chairman',\n",
       " 'challenge',\n",
       " 'champion',\n",
       " 'champion league',\n",
       " 'championship',\n",
       " 'championship series',\n",
       " 'chance',\n",
       " 'change',\n",
       " 'charge',\n",
       " 'charged',\n",
       " 'chart',\n",
       " 'check',\n",
       " 'chelsea',\n",
       " 'chicago',\n",
       " 'chief',\n",
       " 'chief executive',\n",
       " 'child',\n",
       " 'china',\n",
       " 'chinese',\n",
       " 'chip',\n",
       " 'choice',\n",
       " 'christmas',\n",
       " 'cingular',\n",
       " 'cisco',\n",
       " 'citing',\n",
       " 'city',\n",
       " 'civil',\n",
       " 'civilian',\n",
       " 'claim',\n",
       " 'claimed',\n",
       " 'clash',\n",
       " 'class',\n",
       " 'classic',\n",
       " 'clear',\n",
       " 'cleric',\n",
       " 'cleveland',\n",
       " 'client',\n",
       " 'climate',\n",
       " 'close',\n",
       " 'closed',\n",
       " 'closer',\n",
       " 'closing',\n",
       " 'club',\n",
       " 'co',\n",
       " 'coach',\n",
       " 'coalition',\n",
       " 'coast',\n",
       " 'code',\n",
       " 'colin',\n",
       " 'college',\n",
       " 'color',\n",
       " 'colorado',\n",
       " 'colt',\n",
       " 'come',\n",
       " 'coming',\n",
       " 'comment',\n",
       " 'commerce',\n",
       " 'commercial',\n",
       " 'commission',\n",
       " 'committee',\n",
       " 'common',\n",
       " 'communication',\n",
       " 'community',\n",
       " 'company',\n",
       " 'competition',\n",
       " 'complete',\n",
       " 'computer',\n",
       " 'computing',\n",
       " 'concern',\n",
       " 'condition',\n",
       " 'conference',\n",
       " 'confidence',\n",
       " 'confirmed',\n",
       " 'conflict',\n",
       " 'congress',\n",
       " 'connection',\n",
       " 'consecutive',\n",
       " 'consider',\n",
       " 'considering',\n",
       " 'consumer',\n",
       " 'content',\n",
       " 'continue',\n",
       " 'continued',\n",
       " 'continues',\n",
       " 'contract',\n",
       " 'control',\n",
       " 'controversial',\n",
       " 'convention',\n",
       " 'core',\n",
       " 'corp',\n",
       " 'corp target',\n",
       " 'corporate',\n",
       " 'corporation',\n",
       " 'cost',\n",
       " 'could',\n",
       " 'council',\n",
       " 'country',\n",
       " 'county',\n",
       " 'course',\n",
       " 'court',\n",
       " 'cp',\n",
       " 'crash',\n",
       " 'create',\n",
       " 'created',\n",
       " 'credit',\n",
       " 'crew',\n",
       " 'cricket',\n",
       " 'crime',\n",
       " 'criminal',\n",
       " 'crisis',\n",
       " 'critical',\n",
       " 'cross',\n",
       " 'crowd',\n",
       " 'crucial',\n",
       " 'crude',\n",
       " 'crude oil',\n",
       " 'cub',\n",
       " 'cuba',\n",
       " 'cup',\n",
       " 'currency',\n",
       " 'current',\n",
       " 'customer',\n",
       " 'cut',\n",
       " 'cut job',\n",
       " 'cutting',\n",
       " 'daily',\n",
       " 'dallas',\n",
       " 'damage',\n",
       " 'dame',\n",
       " 'darfur',\n",
       " 'darfur region',\n",
       " 'data',\n",
       " 'date',\n",
       " 'david',\n",
       " 'davis',\n",
       " 'day',\n",
       " 'de',\n",
       " 'dead',\n",
       " 'deadline',\n",
       " 'deadly',\n",
       " 'deal',\n",
       " 'death',\n",
       " 'debate',\n",
       " 'debt',\n",
       " 'debut',\n",
       " 'dec',\n",
       " 'decade',\n",
       " 'december',\n",
       " 'decided',\n",
       " 'decision',\n",
       " 'declared',\n",
       " 'decline',\n",
       " 'deep',\n",
       " 'defeat',\n",
       " 'defence',\n",
       " 'defending',\n",
       " 'defense',\n",
       " 'defensive',\n",
       " 'deficit',\n",
       " 'delay',\n",
       " 'delayed',\n",
       " 'delhi',\n",
       " 'dell',\n",
       " 'delta',\n",
       " 'demand',\n",
       " 'democracy',\n",
       " 'democrat',\n",
       " 'democratic',\n",
       " 'department',\n",
       " 'design',\n",
       " 'designed',\n",
       " 'desktop',\n",
       " 'desktop search',\n",
       " 'despite',\n",
       " 'detail',\n",
       " 'detroit',\n",
       " 'develop',\n",
       " 'developed',\n",
       " 'developer',\n",
       " 'developing',\n",
       " 'development',\n",
       " 'device',\n",
       " 'died',\n",
       " 'diego',\n",
       " 'different',\n",
       " 'digital',\n",
       " 'diplomat',\n",
       " 'direct',\n",
       " 'director',\n",
       " 'disaster',\n",
       " 'disc',\n",
       " 'discovered',\n",
       " 'discus',\n",
       " 'disease',\n",
       " 'disney',\n",
       " 'display',\n",
       " 'dispute',\n",
       " 'district',\n",
       " 'division',\n",
       " 'doctor',\n",
       " 'document',\n",
       " 'dodger',\n",
       " 'dollar',\n",
       " 'dollar barrel',\n",
       " 'dollar billion',\n",
       " 'dollar million',\n",
       " 'dollarm',\n",
       " 'dolphin',\n",
       " 'domestic',\n",
       " 'done',\n",
       " 'double',\n",
       " 'doubt',\n",
       " 'dow',\n",
       " 'dow jones',\n",
       " 'download',\n",
       " 'dozen',\n",
       " 'draft',\n",
       " 'draw',\n",
       " 'dream',\n",
       " 'drive',\n",
       " 'driver',\n",
       " 'drop',\n",
       " 'dropped',\n",
       " 'drug',\n",
       " 'dual',\n",
       " 'due',\n",
       " 'dutch',\n",
       " 'dvd',\n",
       " 'eagle',\n",
       " 'earlier',\n",
       " 'early',\n",
       " 'earnings',\n",
       " 'earth',\n",
       " 'earthquake',\n",
       " 'east',\n",
       " 'eastern',\n",
       " 'easy',\n",
       " 'ebay',\n",
       " 'economic',\n",
       " 'economic growth',\n",
       " 'economy',\n",
       " 'edge',\n",
       " 'edition',\n",
       " 'edward',\n",
       " 'effect',\n",
       " 'effort',\n",
       " 'egypt',\n",
       " 'egyptian',\n",
       " 'eight',\n",
       " 'el',\n",
       " 'election',\n",
       " 'electronic',\n",
       " 'electronics',\n",
       " 'embassy',\n",
       " 'emergency',\n",
       " 'employee',\n",
       " 'end',\n",
       " 'ended',\n",
       " 'ending',\n",
       " 'energy',\n",
       " 'engine',\n",
       " 'engineer',\n",
       " 'england',\n",
       " 'english',\n",
       " 'enough',\n",
       " 'enrichment',\n",
       " 'enron',\n",
       " 'enter',\n",
       " 'enterprise',\n",
       " 'entertainment',\n",
       " 'equipment',\n",
       " 'estimate',\n",
       " 'eu',\n",
       " 'euro',\n",
       " 'europe',\n",
       " 'european',\n",
       " 'european union',\n",
       " 'even',\n",
       " 'event',\n",
       " 'ever',\n",
       " 'every',\n",
       " 'evidence',\n",
       " 'ex',\n",
       " 'exchange',\n",
       " 'exchange commission',\n",
       " 'executive',\n",
       " 'expand',\n",
       " 'expansion',\n",
       " 'expectation',\n",
       " 'expected',\n",
       " 'expects',\n",
       " 'expert',\n",
       " 'exploded',\n",
       " 'explorer',\n",
       " 'explosion',\n",
       " 'explosive',\n",
       " 'expo',\n",
       " 'export',\n",
       " 'express',\n",
       " 'extend',\n",
       " 'extended',\n",
       " 'extension',\n",
       " 'extra',\n",
       " 'eye',\n",
       " 'face',\n",
       " 'face verdana',\n",
       " 'facing',\n",
       " 'factory',\n",
       " 'failed',\n",
       " 'failure',\n",
       " 'fall',\n",
       " 'falling',\n",
       " 'falluja',\n",
       " 'fallujah',\n",
       " 'familiar',\n",
       " 'family',\n",
       " 'fan',\n",
       " 'fannie',\n",
       " 'fannie mae',\n",
       " 'far',\n",
       " 'fast',\n",
       " 'faster',\n",
       " 'fastest',\n",
       " 'fear',\n",
       " 'feature',\n",
       " 'fed',\n",
       " 'federal',\n",
       " 'federal reserve',\n",
       " 'federer',\n",
       " 'fee',\n",
       " 'fell',\n",
       " 'fi',\n",
       " 'field',\n",
       " 'fifth',\n",
       " 'fight',\n",
       " 'fighter',\n",
       " 'fighting',\n",
       " 'figure',\n",
       " 'file',\n",
       " 'filed',\n",
       " 'filing',\n",
       " 'film',\n",
       " 'final',\n",
       " 'finally',\n",
       " 'finance',\n",
       " 'financial',\n",
       " 'find',\n",
       " 'finding',\n",
       " 'fine',\n",
       " 'finish',\n",
       " 'fire',\n",
       " 'fired',\n",
       " 'firefox',\n",
       " 'firm',\n",
       " 'first',\n",
       " 'first half',\n",
       " 'first round',\n",
       " 'first time',\n",
       " 'fiscal',\n",
       " 'five',\n",
       " 'five year',\n",
       " 'fix',\n",
       " 'fla',\n",
       " 'flat',\n",
       " 'flaw',\n",
       " 'flight',\n",
       " 'flood',\n",
       " 'florida',\n",
       " 'flu',\n",
       " 'fly',\n",
       " 'focus',\n",
       " 'following',\n",
       " 'font',\n",
       " 'font face',\n",
       " 'food',\n",
       " 'foot',\n",
       " 'football',\n",
       " 'force',\n",
       " 'forced',\n",
       " 'ford',\n",
       " 'forecast',\n",
       " 'foreign',\n",
       " 'foreign minister',\n",
       " 'forest',\n",
       " 'form',\n",
       " 'format',\n",
       " 'former',\n",
       " 'formula',\n",
       " 'formula one',\n",
       " 'forward',\n",
       " 'found',\n",
       " 'founder',\n",
       " 'four',\n",
       " 'four year',\n",
       " 'fourth',\n",
       " 'fourth quarter',\n",
       " 'france',\n",
       " 'francisco',\n",
       " 'fraud',\n",
       " 'free',\n",
       " 'freed',\n",
       " 'french',\n",
       " 'fresh',\n",
       " 'friday',\n",
       " 'friday night',\n",
       " 'friend',\n",
       " 'front',\n",
       " 'fuel',\n",
       " 'full',\n",
       " 'fullquote',\n",
       " 'fund',\n",
       " 'future',\n",
       " 'gain',\n",
       " 'game',\n",
       " 'gas',\n",
       " 'gate',\n",
       " 'gave',\n",
       " 'gaza',\n",
       " 'gaza strip',\n",
       " 'general',\n",
       " 'generation',\n",
       " 'george',\n",
       " 'george bush',\n",
       " 'georgia',\n",
       " 'german',\n",
       " 'germany',\n",
       " 'get',\n",
       " 'getting',\n",
       " 'giant',\n",
       " 'girl',\n",
       " 'give',\n",
       " 'given',\n",
       " 'giving',\n",
       " 'global',\n",
       " 'gm',\n",
       " 'go',\n",
       " 'goal',\n",
       " 'going',\n",
       " 'gold',\n",
       " 'gold medal',\n",
       " 'golf',\n",
       " 'good',\n",
       " 'google',\n",
       " 'google inc',\n",
       " 'got',\n",
       " 'government',\n",
       " 'governor',\n",
       " 'grand',\n",
       " 'grand prix',\n",
       " 'great',\n",
       " 'greece',\n",
       " 'greek',\n",
       " 'green',\n",
       " 'grew',\n",
       " 'ground',\n",
       " 'group',\n",
       " 'group inc',\n",
       " 'growing',\n",
       " 'growth',\n",
       " 'guard',\n",
       " 'guilty',\n",
       " 'gulf',\n",
       " 'gunman',\n",
       " 'hacker',\n",
       " 'haiti',\n",
       " 'half',\n",
       " 'halt',\n",
       " 'hamas',\n",
       " 'hand',\n",
       " 'handed',\n",
       " 'handheld',\n",
       " 'handset',\n",
       " 'hard',\n",
       " 'hat',\n",
       " 'head',\n",
       " 'headed',\n",
       " 'health',\n",
       " 'hearing',\n",
       " 'heart',\n",
       " 'heat',\n",
       " 'heating',\n",
       " 'heavy',\n",
       " 'held',\n",
       " 'help',\n",
       " 'helped',\n",
       " 'helping',\n",
       " 'helvetica',\n",
       " 'helvetica size',\n",
       " 'hewitt',\n",
       " 'hewlett',\n",
       " 'hewlett packard',\n",
       " 'high',\n",
       " 'high speed',\n",
       " 'higher',\n",
       " 'highest',\n",
       " 'hike',\n",
       " 'hill',\n",
       " 'historic',\n",
       " 'history',\n",
       " 'hit',\n",
       " 'hockey',\n",
       " 'hold',\n",
       " 'holding',\n",
       " 'hole',\n",
       " 'holiday',\n",
       " 'hollywood',\n",
       " 'home',\n",
       " 'homer',\n",
       " 'hong',\n",
       " 'hong kong',\n",
       " 'hope',\n",
       " 'hoping',\n",
       " 'hospital',\n",
       " 'host',\n",
       " 'hostage',\n",
       " 'hostile',\n",
       " 'hot',\n",
       " 'hotel',\n",
       " 'hour',\n",
       " 'house',\n",
       " 'houston',\n",
       " 'howard',\n",
       " 'hp',\n",
       " 'huge',\n",
       " 'human',\n",
       " 'hundred',\n",
       " 'hurricane',\n",
       " 'hurricane ivan',\n",
       " 'hurt',\n",
       " 'ibm',\n",
       " 'id',\n",
       " 'idea',\n",
       " 'illegal',\n",
       " 'image',\n",
       " 'impact',\n",
       " 'important',\n",
       " 'improve',\n",
       " 'inc',\n",
       " 'inc target',\n",
       " 'including',\n",
       " 'income',\n",
       " 'increase',\n",
       " 'increased',\n",
       " 'independent',\n",
       " 'index',\n",
       " 'india',\n",
       " 'indian',\n",
       " 'indianapolis',\n",
       " 'indonesia',\n",
       " 'indonesian',\n",
       " 'industrial',\n",
       " 'industry',\n",
       " 'inflation',\n",
       " 'info',\n",
       " 'information',\n",
       " 'initial',\n",
       " 'injured',\n",
       " 'injury',\n",
       " 'inning',\n",
       " 'inside',\n",
       " 'instead',\n",
       " 'insurance',\n",
       " 'insurer',\n",
       " 'insurgent',\n",
       " 'intel',\n",
       " 'intelligence',\n",
       " 'interest',\n",
       " 'interest rate',\n",
       " 'interim',\n",
       " 'international',\n",
       " 'internet',\n",
       " 'internet explorer',\n",
       " 'interview',\n",
       " 'introduced',\n",
       " 'inventory',\n",
       " 'investigation',\n",
       " 'investment',\n",
       " 'investor',\n",
       " 'ipo',\n",
       " 'ipod',\n",
       " 'iran',\n",
       " 'iraq',\n",
       " 'iraqi',\n",
       " 'ireland',\n",
       " 'irish',\n",
       " 'islamic',\n",
       " 'island',\n",
       " 'israel',\n",
       " 'israeli',\n",
       " 'issue',\n",
       " 'issued',\n",
       " 'italian',\n",
       " 'italy',\n",
       " 'itunes',\n",
       " 'ivan',\n",
       " 'ivory',\n",
       " 'ivory coast',\n",
       " 'jail',\n",
       " 'jakarta',\n",
       " 'james',\n",
       " 'january',\n",
       " 'japan',\n",
       " 'japanese',\n",
       " 'jason',\n",
       " 'java',\n",
       " 'jay',\n",
       " 'jerusalem',\n",
       " 'jet',\n",
       " 'job',\n",
       " 'john',\n",
       " 'john kerry',\n",
       " 'johnson',\n",
       " 'join',\n",
       " 'joint',\n",
       " 'jones',\n",
       " 'jose',\n",
       " 'journalist',\n",
       " 'judge',\n",
       " 'july',\n",
       " 'jump',\n",
       " 'june',\n",
       " 'justice',\n",
       " 'kabul',\n",
       " 'kansa',\n",
       " 'karzai',\n",
       " 'kashmir',\n",
       " 'keep',\n",
       " 'kept',\n",
       " 'kerry',\n",
       " 'key',\n",
       " 'kidnapped',\n",
       " 'kill',\n",
       " 'killed',\n",
       " 'killing',\n",
       " 'killing least',\n",
       " 'king',\n",
       " 'knee',\n",
       " 'know',\n",
       " 'known',\n",
       " 'kong',\n",
       " 'korea',\n",
       " 'korean',\n",
       " 'la',\n",
       " 'labor',\n",
       " 'land',\n",
       " 'large',\n",
       " 'largest',\n",
       " 'last',\n",
       " 'last month',\n",
       " 'last night',\n",
       " 'last week',\n",
       " 'last year',\n",
       " 'late',\n",
       " 'later',\n",
       " 'latest',\n",
       " 'launch',\n",
       " 'launched',\n",
       " 'law',\n",
       " 'lawsuit',\n",
       " 'lawyer',\n",
       " 'le',\n",
       " 'lead',\n",
       " 'leader',\n",
       " 'leading',\n",
       " 'leaf',\n",
       " 'league',\n",
       " 'least',\n",
       " 'least people',\n",
       " 'leave',\n",
       " 'leaving',\n",
       " 'led',\n",
       " 'left',\n",
       " 'legal',\n",
       " 'let',\n",
       " 'level',\n",
       " 'life',\n",
       " 'lift',\n",
       " 'light',\n",
       " 'like',\n",
       " 'likely',\n",
       " 'limit',\n",
       " 'line',\n",
       " 'link',\n",
       " 'linked',\n",
       " 'linux',\n",
       " 'lion',\n",
       " 'list',\n",
       " 'little',\n",
       " 'live',\n",
       " 'liverpool',\n",
       " 'living',\n",
       " 'local',\n",
       " 'london',\n",
       " 'london reuters',\n",
       " 'long',\n",
       " 'longer',\n",
       " 'look',\n",
       " 'looked',\n",
       " 'looking',\n",
       " 'los',\n",
       " 'los angeles',\n",
       " 'lose',\n",
       " 'losing',\n",
       " 'loss',\n",
       " 'lost',\n",
       " 'lot',\n",
       " 'louis',\n",
       " 'love',\n",
       " 'low',\n",
       " 'lower',\n",
       " 'ltd',\n",
       " 'mac',\n",
       " 'machine',\n",
       " 'made',\n",
       " 'madrid',\n",
       " 'mae',\n",
       " 'mail',\n",
       " 'main',\n",
       " 'major',\n",
       " ...]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_tf.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'analyzer': 'word',\n",
       " 'binary': False,\n",
       " 'decode_error': 'strict',\n",
       " 'dtype': numpy.float64,\n",
       " 'encoding': 'utf-8',\n",
       " 'input': 'content',\n",
       " 'lowercase': False,\n",
       " 'max_df': 0.75,\n",
       " 'max_features': 2000,\n",
       " 'min_df': 1,\n",
       " 'ngram_range': (1, 2),\n",
       " 'norm': 'l2',\n",
       " 'preprocessor': None,\n",
       " 'smooth_idf': True,\n",
       " 'stop_words': None,\n",
       " 'strip_accents': None,\n",
       " 'sublinear_tf': False,\n",
       " 'token_pattern': '(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       " 'tokenizer': None,\n",
       " 'use_idf': True,\n",
       " 'vocabulary': None}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_tf.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class Index</th>\n",
       "      <th>Summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>['wall', 'st', 'bear', 'claw', 'back', 'black'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>['carlyle', 'look', 'toward', 'commercial', 'a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>['oil', 'economy', 'cloud', 'stock', 'outlook'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>['iraq', 'halt', 'oil', 'export', 'main', 'sou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>['oil', 'price', 'soar', 'time', 'record', 'po...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Class Index                                            Summary\n",
       "0            3  ['wall', 'st', 'bear', 'claw', 'back', 'black'...\n",
       "1            3  ['carlyle', 'look', 'toward', 'commercial', 'a...\n",
       "2            3  ['oil', 'economy', 'cloud', 'stock', 'outlook'...\n",
       "3            3  ['iraq', 'halt', 'oil', 'export', 'main', 'sou...\n",
       "4            3  ['oil', 'price', 'soar', 'time', 'record', 'po..."
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>able</th>\n",
       "      <th>abu</th>\n",
       "      <th>abuse</th>\n",
       "      <th>access</th>\n",
       "      <th>according</th>\n",
       "      <th>account</th>\n",
       "      <th>accounting</th>\n",
       "      <th>accused</th>\n",
       "      <th>acquire</th>\n",
       "      <th>acquisition</th>\n",
       "      <th>...</th>\n",
       "      <th>yet</th>\n",
       "      <th>york</th>\n",
       "      <th>york reuters</th>\n",
       "      <th>york yankee</th>\n",
       "      <th>young</th>\n",
       "      <th>yukos</th>\n",
       "      <th>zealand</th>\n",
       "      <th>zimbabwe</th>\n",
       "      <th>zone</th>\n",
       "      <th>Class Index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.269783</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.115351</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81659</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.195097</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81660</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81661</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.173916</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81662</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81663</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>81664 rows × 2001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       able  abu  abuse  access  according   account  accounting   accused  \\\n",
       "0       0.0  0.0    0.0     0.0   0.000000  0.000000         0.0  0.000000   \n",
       "1       0.0  0.0    0.0     0.0   0.000000  0.000000         0.0  0.269783   \n",
       "2       0.0  0.0    0.0     0.0   0.000000  0.000000         0.0  0.000000   \n",
       "3       0.0  0.0    0.0     0.0   0.000000  0.000000         0.0  0.000000   \n",
       "4       0.0  0.0    0.0     0.0   0.000000  0.115351         0.0  0.000000   \n",
       "...     ...  ...    ...     ...        ...       ...         ...       ...   \n",
       "81659   0.0  0.0    0.0     0.0   0.195097  0.000000         0.0  0.000000   \n",
       "81660   0.0  0.0    0.0     0.0   0.000000  0.000000         0.0  0.000000   \n",
       "81661   0.0  0.0    0.0     0.0   0.173916  0.000000         0.0  0.000000   \n",
       "81662   0.0  0.0    0.0     0.0   0.000000  0.000000         0.0  0.000000   \n",
       "81663   0.0  0.0    0.0     0.0   0.000000  0.000000         0.0  0.000000   \n",
       "\n",
       "       acquire  acquisition  ...  yet  york  york reuters  york yankee  young  \\\n",
       "0          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "1          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "2          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "3          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "4          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "...        ...          ...  ...  ...   ...           ...          ...    ...   \n",
       "81659      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "81660      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "81661      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "81662      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "81663      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "\n",
       "       yukos  zealand  zimbabwe  zone  Class Index  \n",
       "0        0.0      0.0       0.0   0.0            1  \n",
       "1        0.0      0.0       0.0   0.0            1  \n",
       "2        0.0      0.0       0.0   0.0            3  \n",
       "3        0.0      0.0       0.0   0.0            4  \n",
       "4        0.0      0.0       0.0   0.0            4  \n",
       "...      ...      ...       ...   ...          ...  \n",
       "81659    0.0      0.0       0.0   0.0            3  \n",
       "81660    0.0      0.0       0.0   0.0            1  \n",
       "81661    0.0      0.0       0.0   0.0            3  \n",
       "81662    0.0      0.0       0.0   0.0            4  \n",
       "81663    0.0      0.0       0.0   0.0            1  \n",
       "\n",
       "[81664 rows x 2001 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.DataFrame(train_vectors.toarray(),columns=vectorizer_tf.get_feature_names())\n",
    "train_df = pd.concat([train_df,X_train['Class Index'].reset_index(drop = True)],axis = 1)\n",
    "\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>able</th>\n",
       "      <th>abu</th>\n",
       "      <th>abuse</th>\n",
       "      <th>access</th>\n",
       "      <th>according</th>\n",
       "      <th>account</th>\n",
       "      <th>accounting</th>\n",
       "      <th>accused</th>\n",
       "      <th>acquire</th>\n",
       "      <th>acquisition</th>\n",
       "      <th>...</th>\n",
       "      <th>yet</th>\n",
       "      <th>york</th>\n",
       "      <th>york reuters</th>\n",
       "      <th>york yankee</th>\n",
       "      <th>young</th>\n",
       "      <th>yukos</th>\n",
       "      <th>zealand</th>\n",
       "      <th>zimbabwe</th>\n",
       "      <th>zone</th>\n",
       "      <th>Class Index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.180698</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25515</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25516</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.129202</td>\n",
       "      <td>0.160351</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25517</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25518</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25519</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25520 rows × 2001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       able  abu  abuse  access  according  account  accounting  accused  \\\n",
       "0       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "1       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "2       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "3       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "4       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "...     ...  ...    ...     ...        ...      ...         ...      ...   \n",
       "25515   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "25516   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "25517   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "25518   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "25519   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "\n",
       "       acquire  acquisition  ...  yet      york  york reuters  york yankee  \\\n",
       "0          0.0          0.0  ...  0.0  0.000000      0.000000          0.0   \n",
       "1          0.0          0.0  ...  0.0  0.000000      0.000000          0.0   \n",
       "2          0.0          0.0  ...  0.0  0.000000      0.000000          0.0   \n",
       "3          0.0          0.0  ...  0.0  0.000000      0.000000          0.0   \n",
       "4          0.0          0.0  ...  0.0  0.000000      0.000000          0.0   \n",
       "...        ...          ...  ...  ...       ...           ...          ...   \n",
       "25515      0.0          0.0  ...  0.0  0.000000      0.000000          0.0   \n",
       "25516      0.0          0.0  ...  0.0  0.129202      0.160351          0.0   \n",
       "25517      0.0          0.0  ...  0.0  0.000000      0.000000          0.0   \n",
       "25518      0.0          0.0  ...  0.0  0.000000      0.000000          0.0   \n",
       "25519      0.0          0.0  ...  0.0  0.000000      0.000000          0.0   \n",
       "\n",
       "       young  yukos  zealand  zimbabwe      zone  Class Index  \n",
       "0        0.0    0.0      0.0       0.0  0.000000            1  \n",
       "1        0.0    0.0      0.0       0.0  0.180698            1  \n",
       "2        0.0    0.0      0.0       0.0  0.000000            1  \n",
       "3        0.0    0.0      0.0       0.0  0.000000            4  \n",
       "4        0.0    0.0      0.0       0.0  0.000000            3  \n",
       "...      ...    ...      ...       ...       ...          ...  \n",
       "25515    0.0    0.0      0.0       0.0  0.000000            1  \n",
       "25516    0.0    0.0      0.0       0.0  0.000000            3  \n",
       "25517    0.0    0.0      0.0       0.0  0.000000            3  \n",
       "25518    0.0    0.0      0.0       0.0  0.000000            3  \n",
       "25519    0.0    0.0      0.0       0.0  0.000000            3  \n",
       "\n",
       "[25520 rows x 2001 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.DataFrame(test_vectors.toarray(),columns=vectorizer_tf.get_feature_names())\n",
    "test_df = pd.concat([test_df,X_test['Class Index'].reset_index(drop = True)],axis = 1)\n",
    "\n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>able</th>\n",
       "      <th>abu</th>\n",
       "      <th>abuse</th>\n",
       "      <th>access</th>\n",
       "      <th>according</th>\n",
       "      <th>account</th>\n",
       "      <th>accounting</th>\n",
       "      <th>accused</th>\n",
       "      <th>acquire</th>\n",
       "      <th>acquisition</th>\n",
       "      <th>...</th>\n",
       "      <th>yet</th>\n",
       "      <th>york</th>\n",
       "      <th>york reuters</th>\n",
       "      <th>york yankee</th>\n",
       "      <th>young</th>\n",
       "      <th>yukos</th>\n",
       "      <th>zealand</th>\n",
       "      <th>zimbabwe</th>\n",
       "      <th>zone</th>\n",
       "      <th>Class Index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.560624</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.478845</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20411</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20412</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20413</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20414</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20415</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20416 rows × 2001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       able  abu  abuse  access  according  account  accounting  accused  \\\n",
       "0       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "1       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "2       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "3       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "4       0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "...     ...  ...    ...     ...        ...      ...         ...      ...   \n",
       "20411   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "20412   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "20413   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "20414   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "20415   0.0  0.0    0.0     0.0        0.0      0.0         0.0      0.0   \n",
       "\n",
       "       acquire  acquisition  ...  yet  york  york reuters  york yankee  young  \\\n",
       "0          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "1          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "2          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "3          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "4          0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "...        ...          ...  ...  ...   ...           ...          ...    ...   \n",
       "20411      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "20412      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "20413      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "20414      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "20415      0.0          0.0  ...  0.0   0.0           0.0          0.0    0.0   \n",
       "\n",
       "       yukos   zealand  zimbabwe  zone  Class Index  \n",
       "0        0.0  0.000000  0.560624   0.0            2  \n",
       "1        0.0  0.000000  0.000000   0.0            4  \n",
       "2        0.0  0.478845  0.000000   0.0            2  \n",
       "3        0.0  0.000000  0.000000   0.0            1  \n",
       "4        0.0  0.000000  0.000000   0.0            2  \n",
       "...      ...       ...       ...   ...          ...  \n",
       "20411    0.0  0.000000  0.000000   0.0            1  \n",
       "20412    0.0  0.000000  0.000000   0.0            2  \n",
       "20413    0.0  0.000000  0.000000   0.0            3  \n",
       "20414    0.0  0.000000  0.000000   0.0            1  \n",
       "20415    0.0  0.000000  0.000000   0.0            4  \n",
       "\n",
       "[20416 rows x 2001 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df = pd.DataFrame(val_vectors.toarray(),columns=vectorizer_tf.get_feature_names())\n",
    "val_df = pd.concat([val_df,X_val['Class Index'].reset_index(drop = True)],axis = 1)\n",
    "\n",
    "val_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exporting File to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_csv('train_df.csv',index ='False')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.to_csv('test_df.csv',index ='False')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df.to_csv('val_df.csv',index ='False')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}