bbc copy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from string import digits\n",
    "import requests\n",
    "import re\n",
    "import nltk\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from keras.preprocessing.text import Tokenizer \n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Concatenate, TimeDistributed\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from sklearn.decomposition import LatentDirichletAllocation, TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from bs4 import BeautifulSoup\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Attention\n",
    "import spacy\n",
    "import gensim\n",
    "import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File_path</th>\n",
       "      <th>Articles</th>\n",
       "      <th>Summaries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>business/001.txt</td>\n",
       "      <td>Ad sales boost Time Warner profit..Quarterly p...</td>\n",
       "      <td>TimeWarner said fourth quarter sales rose 2% t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>business/002.txt</td>\n",
       "      <td>Dollar gains on Greenspan speech..The dollar h...</td>\n",
       "      <td>The dollar has hit its highest level against t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>business/003.txt</td>\n",
       "      <td>Yukos unit buyer faces loan claim..The owners ...</td>\n",
       "      <td>Yukos' owner Menatep Group says it will ask Ro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>business/004.txt</td>\n",
       "      <td>High fuel prices hit BA's profits..British Air...</td>\n",
       "      <td>Rod Eddington, BA's chief executive, said the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>business/005.txt</td>\n",
       "      <td>Pernod takeover talk lifts Domecq..Shares in U...</td>\n",
       "      <td>Pernod has reduced the debt it took on to fund...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          File_path                                           Articles  \\\n",
       "0  business/001.txt  Ad sales boost Time Warner profit..Quarterly p...   \n",
       "1  business/002.txt  Dollar gains on Greenspan speech..The dollar h...   \n",
       "2  business/003.txt  Yukos unit buyer faces loan claim..The owners ...   \n",
       "3  business/004.txt  High fuel prices hit BA's profits..British Air...   \n",
       "4  business/005.txt  Pernod takeover talk lifts Domecq..Shares in U...   \n",
       "\n",
       "                                           Summaries  \n",
       "0  TimeWarner said fourth quarter sales rose 2% t...  \n",
       "1  The dollar has hit its highest level against t...  \n",
       "2  Yukos' owner Menatep Group says it will ask Ro...  \n",
       "3  Rod Eddington, BA's chief executive, said the ...  \n",
       "4  Pernod has reduced the debt it took on to fund...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes = os.listdir('./BBC News Summary/News Articles')\n",
    "Articles_dir = './BBC News Summary/News Articles/'\n",
    "Summaries_dir = './BBC News Summary/Summaries/'\n",
    "\n",
    "articles = []\n",
    "summaries = []\n",
    "file_arr = []\n",
    "for cls in classes:\n",
    "    files = os.listdir(Articles_dir + cls)\n",
    "    for file in files:\n",
    "        article_file_path = Articles_dir + cls + '/' + file\n",
    "        summary_file_path = Summaries_dir + cls + '/' + file\n",
    "        try:\n",
    "            with open(article_file_path, 'r') as f:\n",
    "                articles.append('.'.join(\n",
    "                    [line.rstrip() for line in f.readlines()]))\n",
    "            with open(summary_file_path, 'r') as f:\n",
    "                summaries.append('.'.join(\n",
    "                    [line.rstrip() for line in f.readlines()]))\n",
    "            file_arr.append(cls + '/' + file)\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "data = pd.DataFrame({\n",
    "    'File_path': file_arr,\n",
    "    'Articles': articles,\n",
    "    'Summaries': summaries\n",
    "})\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2225 entries, 0 to 2224\n",
      "Data columns (total 3 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   File_path  2225 non-null   object\n",
      " 1   Articles   2225 non-null   object\n",
      " 2   Summaries  2225 non-null   object\n",
      "dtypes: object(3)\n",
      "memory usage: 52.3+ KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File_path</th>\n",
       "      <th>Articles</th>\n",
       "      <th>Summaries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2225</td>\n",
       "      <td>2225</td>\n",
       "      <td>2225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>2225</td>\n",
       "      <td>2127</td>\n",
       "      <td>2081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>business/001.txt</td>\n",
       "      <td>Ray DVD beats box office takings..Oscar-nomina...</td>\n",
       "      <td>Although the two partially-paralysed people pe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               File_path                                           Articles  \\\n",
       "count               2225                                               2225   \n",
       "unique              2225                                               2127   \n",
       "top     business/001.txt  Ray DVD beats box office takings..Oscar-nomina...   \n",
       "freq                   1                                                  2   \n",
       "\n",
       "                                                Summaries  \n",
       "count                                                2225  \n",
       "unique                                               2081  \n",
       "top     Although the two partially-paralysed people pe...  \n",
       "freq                                                    2  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data.to_csv('news.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2225, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "contractions_dictionary = {\n",
    "    \"ain't\": \"is not\",\n",
    "    \"aren't\": \"are not\",\n",
    "    \"can't\": \"cannot\",\n",
    "    \"'cause\": \"because\",\n",
    "    \"could've\": \"could have\",\n",
    "    \"couldn't\": \"could not\",\n",
    "    \"didn't\": \"did not\",\n",
    "    \"doesn't\": \"does not\",\n",
    "    \"don't\": \"do not\",\n",
    "    \"hadn't\": \"had not\",\n",
    "    \"hasn't\": \"has not\",\n",
    "    \"haven't\": \"have not\",\n",
    "    \"he'd\": \"he would\",\n",
    "    \"he'll\": \"he will\",\n",
    "    \"he's\": \"he is\",\n",
    "    \"how'd\": \"how did\",\n",
    "    \"how'd'y\": \"how do you\",\n",
    "    \"how'll\": \"how will\",\n",
    "    \"how's\": \"how is\",\n",
    "    \"I'd\": \"I would\",\n",
    "    \"I'd've\": \"I would have\",\n",
    "    \"I'll\": \"I will\",\n",
    "    \"I'll've\": \"I will have\",\n",
    "    \"I'm\": \"I am\",\n",
    "    \"I've\": \"I have\",\n",
    "    \"i'd\": \"i would\",\n",
    "    \"i'd've\": \"i would have\",\n",
    "    \"i'll\": \"i will\",\n",
    "    \"i'll've\": \"i will have\",\n",
    "    \"i'm\": \"i am\",\n",
    "    \"i've\": \"i have\",\n",
    "    \"isn't\": \"is not\",\n",
    "    \"it'd\": \"it would\",\n",
    "    \"it'd've\": \"it would have\",\n",
    "    \"it'll\": \"it will\",\n",
    "    \"it'll've\": \"it will have\",\n",
    "    \"it's\": \"it is\",\n",
    "    \"let's\": \"let us\",\n",
    "    \"ma'am\": \"madam\",\n",
    "    \"mayn't\": \"may not\",\n",
    "    \"might've\": \"might have\",\n",
    "    \"mightn't\": \"might not\",\n",
    "    \"mightn't've\": \"might not have\",\n",
    "    \"must've\": \"must have\",\n",
    "    \"mustn't\": \"must not\",\n",
    "    \"mustn't've\": \"must not have\",\n",
    "    \"needn't\": \"need not\",\n",
    "    \"needn't've\": \"need not have\",\n",
    "    \"o'clock\": \"of the clock\",\n",
    "    \"oughtn't\": \"ought not\",\n",
    "    \"oughtn't've\": \"ought not have\",\n",
    "    \"shan't\": \"shall not\",\n",
    "    \"sha'n't\": \"shall not\",\n",
    "    \"shan't've\": \"shall not have\",\n",
    "    \"she'd\": \"she would\",\n",
    "    \"she'd've\": \"she would have\",\n",
    "    \"she'll\": \"she will\",\n",
    "    \"she'll've\": \"she will have\",\n",
    "    \"she's\": \"she is\",\n",
    "    \"should've\": \"should have\",\n",
    "    \"shouldn't\": \"should not\",\n",
    "    \"shouldn't've\": \"should not have\",\n",
    "    \"so've\": \"so have\",\n",
    "    \"so's\": \"so as\",\n",
    "    \"this's\": \"this is\",\n",
    "    \"that'd\": \"that would\",\n",
    "    \"that'd've\": \"that would have\",\n",
    "    \"that's\": \"that is\",\n",
    "    \"there'd\": \"there would\",\n",
    "    \"there'd've\": \"there would have\",\n",
    "    \"there's\": \"there is\",\n",
    "    \"here's\": \"here is\",\n",
    "    \"they'd\": \"they would\",\n",
    "    \"they'd've\": \"they would have\",\n",
    "    \"they'll\": \"they will\",\n",
    "    \"they'll've\": \"they will have\",\n",
    "    \"they're\": \"they are\",\n",
    "    \"they've\": \"they have\",\n",
    "    \"to've\": \"to have\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'd\": \"we would\",\n",
    "    \"we'd've\": \"we would have\",\n",
    "    \"we'll\": \"we will\",\n",
    "    \"we'll've\": \"we will have\",\n",
    "    \"we're\": \"we are\",\n",
    "    \"we've\": \"we have\",\n",
    "    \"weren't\": \"were not\",\n",
    "    \"what'll\": \"what will\",\n",
    "    \"what'll've\": \"what will have\",\n",
    "    \"what're\": \"what are\",\n",
    "    \"what's\": \"what is\",\n",
    "    \"what've\": \"what have\",\n",
    "    \"when's\": \"when is\",\n",
    "    \"when've\": \"when have\",\n",
    "    \"where'd\": \"where did\",\n",
    "    \"where's\": \"where is\",\n",
    "    \"where've\": \"where have\",\n",
    "    \"who'll\": \"who will\",\n",
    "    \"who'll've\": \"who will have\",\n",
    "    \"who's\": \"who is\",\n",
    "    \"who've\": \"who have\",\n",
    "    \"why's\": \"why is\",\n",
    "    \"why've\": \"why have\",\n",
    "    \"will've\": \"will have\",\n",
    "    \"won't\": \"will not\",\n",
    "    \"won't've\": \"will not have\",\n",
    "    \"would've\": \"would have\",\n",
    "    \"wouldn't\": \"would not\",\n",
    "    \"wouldn't've\": \"would not have\",\n",
    "    \"y'all\": \"you all\",\n",
    "    \"y'all'd\": \"you all would\",\n",
    "    \"y'all'd've\": \"you all would have\",\n",
    "    \"y'all're\": \"you all are\",\n",
    "    \"y'all've\": \"you all have\",\n",
    "    \"you'd\": \"you would\",\n",
    "    \"you'd've\": \"you would have\",\n",
    "    \"you'll\": \"you will\",\n",
    "    \"you'll've\": \"you will have\",\n",
    "    \"you're\": \"you are\",\n",
    "    \"you've\": \"you have\"\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Filter(text):\n",
    "    text = text.lower()\n",
    "    text = ' '.join([\n",
    "        contractions_dictionary[i]\n",
    "        if i in contractions_dictionary.keys() else i for i in text.split()\n",
    "    ])\n",
    "    text = re.sub(r'\\(.*\\)', \"\", text)\n",
    "    text = re.sub(\"'s\", \"\", text)\n",
    "    text = re.sub('\"', '', text)\n",
    "    text = ' '.join([i for i in text.split() if i.isalpha()])\n",
    "    text = re.sub('[^a-zA-Z]', \" \", text)\n",
    "\n",
    "    return text\n",
    "\n",
    "\n",
    "data['File_path'] = data['File_path'].apply(Filter)\n",
    "data['Articles'] = data['Articles'].apply(Filter)\n",
    "data['Summaries'] = data['Summaries'].apply(Filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File_path</th>\n",
       "      <th>Articles</th>\n",
       "      <th>Summaries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>ad sales boost time warner profits at us media...</td>\n",
       "      <td>timewarner said fourth quarter sales rose to f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>dollar gains on greenspan dollar has hit its h...</td>\n",
       "      <td>the dollar has hit its highest level against t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>yukos unit buyer faces loan owners of embattle...</td>\n",
       "      <td>owner menatep group says it will ask rosneft t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>high fuel prices hit ba airways has blamed hig...</td>\n",
       "      <td>rod ba chief said the results were respectable...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>pernod takeover talk lifts in uk drinks and fo...</td>\n",
       "      <td>pernod has reduced the debt it took on to fund...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2220</th>\n",
       "      <td></td>\n",
       "      <td>bt program to beat dialler is introducing two ...</td>\n",
       "      <td>bt is introducing two initiatives to help beat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2221</th>\n",
       "      <td></td>\n",
       "      <td>spam tempt net users across the world continue...</td>\n",
       "      <td>a third of them read unsolicited junk and buy ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2222</th>\n",
       "      <td></td>\n",
       "      <td>be careful how you new european directive coul...</td>\n",
       "      <td>this goes to the heart of the european and eve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2223</th>\n",
       "      <td></td>\n",
       "      <td>us cyber security chief man making sure us com...</td>\n",
       "      <td>amit yoran was director of the national cyber ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2224</th>\n",
       "      <td></td>\n",
       "      <td>losing yourself in online role playing games a...</td>\n",
       "      <td>he says that in the world of online gaming suc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2225 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     File_path                                           Articles  \\\n",
       "0               ad sales boost time warner profits at us media...   \n",
       "1               dollar gains on greenspan dollar has hit its h...   \n",
       "2               yukos unit buyer faces loan owners of embattle...   \n",
       "3               high fuel prices hit ba airways has blamed hig...   \n",
       "4               pernod takeover talk lifts in uk drinks and fo...   \n",
       "...        ...                                                ...   \n",
       "2220            bt program to beat dialler is introducing two ...   \n",
       "2221            spam tempt net users across the world continue...   \n",
       "2222            be careful how you new european directive coul...   \n",
       "2223            us cyber security chief man making sure us com...   \n",
       "2224            losing yourself in online role playing games a...   \n",
       "\n",
       "                                              Summaries  \n",
       "0     timewarner said fourth quarter sales rose to f...  \n",
       "1     the dollar has hit its highest level against t...  \n",
       "2     owner menatep group says it will ask rosneft t...  \n",
       "3     rod ba chief said the results were respectable...  \n",
       "4     pernod has reduced the debt it took on to fund...  \n",
       "...                                                 ...  \n",
       "2220  bt is introducing two initiatives to help beat...  \n",
       "2221  a third of them read unsolicited junk and buy ...  \n",
       "2222  this goes to the heart of the european and eve...  \n",
       "2223  amit yoran was director of the national cyber ...  \n",
       "2224  he says that in the world of online gaming suc...  \n",
       "\n",
       "[2225 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data = data.drop(['File_path'], axis=1)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove HTML tags from the data frame if they are present\n",
    "\n",
    "\n",
    "def strip_html(text):\n",
    "    soup = BeautifulSoup(text, \"html.parser\")\n",
    "    return soup.get_text()\n",
    "\n",
    "\n",
    "data['Summary'] = data['Summaries'].apply(strip_html)\n",
    "data['Article'] = data['Articles'].apply(strip_html)\n",
    "\n",
    "data = data.drop(['Summaries', 'Articles'], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File_path</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Article</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>timewarner said fourth quarter sales rose to f...</td>\n",
       "      <td>ad sales boost time warner profits at us media...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>the dollar has hit its highest level against t...</td>\n",
       "      <td>dollar gains on greenspan dollar has hit its h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>owner menatep group says it will ask rosneft t...</td>\n",
       "      <td>yukos unit buyer faces loan owners of embattle...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>rod ba chief said the results were respectable...</td>\n",
       "      <td>high fuel prices hit ba airways has blamed hig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>pernod has reduced the debt it took on to fund...</td>\n",
       "      <td>pernod takeover talk lifts in uk drinks and fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2220</th>\n",
       "      <td></td>\n",
       "      <td>bt is introducing two initiatives to help beat...</td>\n",
       "      <td>bt program to beat dialler is introducing two ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2221</th>\n",
       "      <td></td>\n",
       "      <td>a third of them read unsolicited junk and buy ...</td>\n",
       "      <td>spam tempt net users across the world continue...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2222</th>\n",
       "      <td></td>\n",
       "      <td>this goes to the heart of the european and eve...</td>\n",
       "      <td>be careful how you new european directive coul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2223</th>\n",
       "      <td></td>\n",
       "      <td>amit yoran was director of the national cyber ...</td>\n",
       "      <td>us cyber security chief man making sure us com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2224</th>\n",
       "      <td></td>\n",
       "      <td>he says that in the world of online gaming suc...</td>\n",
       "      <td>losing yourself in online role playing games a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2225 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     File_path                                            Summary  \\\n",
       "0               timewarner said fourth quarter sales rose to f...   \n",
       "1               the dollar has hit its highest level against t...   \n",
       "2               owner menatep group says it will ask rosneft t...   \n",
       "3               rod ba chief said the results were respectable...   \n",
       "4               pernod has reduced the debt it took on to fund...   \n",
       "...        ...                                                ...   \n",
       "2220            bt is introducing two initiatives to help beat...   \n",
       "2221            a third of them read unsolicited junk and buy ...   \n",
       "2222            this goes to the heart of the european and eve...   \n",
       "2223            amit yoran was director of the national cyber ...   \n",
       "2224            he says that in the world of online gaming suc...   \n",
       "\n",
       "                                                Article  \n",
       "0     ad sales boost time warner profits at us media...  \n",
       "1     dollar gains on greenspan dollar has hit its h...  \n",
       "2     yukos unit buyer faces loan owners of embattle...  \n",
       "3     high fuel prices hit ba airways has blamed hig...  \n",
       "4     pernod takeover talk lifts in uk drinks and fo...  \n",
       "...                                                 ...  \n",
       "2220  bt program to beat dialler is introducing two ...  \n",
       "2221  spam tempt net users across the world continue...  \n",
       "2222  be careful how you new european directive coul...  \n",
       "2223  us cyber security chief man making sure us com...  \n",
       "2224  losing yourself in online role playing games a...  \n",
       "\n",
       "[2225 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove Stop Words from the Summary\n",
    "\n",
    "\n",
    "def remove_stopword(text):\n",
    "    stopword = nltk.corpus.stopwords.words('english')\n",
    "    stopword.remove('not')\n",
    "    a = [w for w in nltk.word_tokenize(text) if w not in stopword]\n",
    "    return ' '.join(a)\n",
    "\n",
    "\n",
    "data['Summary'] = data['Summary'].apply(remove_stopword)\n",
    "data['Article'] = data['Article'].apply(remove_stopword)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def punc_clean(text):\n",
    "    import string as st\n",
    "    a = [w for w in text if w not in st.punctuation]\n",
    "    return ''.join(a)\n",
    "\n",
    "\n",
    "data['Summary'] = data['Summary'].apply(punc_clean)\n",
    "data['Article'] = data['Article'].apply(punc_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.Article.values.tolist()\n",
    "\n",
    "# Remove Emails\n",
    "data = [re.sub('\\S*@\\S*\\s?', '', sent) for sent in data]\n",
    "\n",
    "# Remove new line characters\n",
    "data = [re.sub('\\s+', ' ', sent) for sent in data]\n",
    "\n",
    "# Remove distracting single quotes\n",
    "data = [re.sub(\"\\'\", \"\", sent) for sent in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['ad', 'sales', 'boost', 'time', 'warner', 'profits', 'us', 'media', 'giant', 'timewarner', 'jumped', 'close', 'warner', 'fourth', 'quarter', 'profits', 'slightly', 'better', 'film', 'division', 'saw', 'profits', 'slump', 'helped', 'flops', 'alexander', 'sharp', 'contrast', 'third', 'final', 'film', 'lord', 'rings', 'trilogy', 'boosted', 'timewarner', 'posted', 'profit', 'revenues', 'grew', 'financial', 'performance', 'meeting', 'exceeding', 'objectives', 'greatly', 'enhancing', 'chairman', 'chief', 'executive', 'richard', 'parsons', 'timewarner', 'projecting', 'operating', 'earnings', 'growth', 'around', 'also', 'expects', 'higher', 'revenue', 'wider', 'profit', 'restate', 'accounts', 'part', 'efforts', 'resolve', 'inquiry', 'aol', 'us', 'market', 'already', 'offered', 'pay', 'settle', 'deal', 'review', 'company', 'said', 'unable', 'estimate', 'amount', 'needed', 'set', 'aside', 'legal', 'previously', 'set', 'intends', 'adjust', 'way', 'accounts', 'deal', 'german', 'music', 'publisher', 'bertelsmann', 'purchase', 'stake', 'aol', 'reported', 'advertising', 'book', 'sale', 'stake', 'aol', 'europe', 'loss', 'value']]\n"
     ]
    }
   ],
   "source": [
    "def sent_to_words(sentences, deacc=True): \n",
    "    for sentence in sentences:\n",
    "        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))\n",
    "        \n",
    "data_words = list(sent_to_words(data))     \n",
    "\n",
    "print(data_words[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ad sale boost time profit medium giant timewarner jump close warner fourth quarter profit slightly well film division see profit slump help flop alexander sharp contrast third final film trilogy boost timewarner post profit revenue grow financial performance meeting exceed objective greatly enhance chairman chief executive timewarner project operate earning growth around also expect high revenue wide profit restate account part effort resolve inquiry market already offer pay settle deal review company say unable estimate amount need set aside legal previously set intend adjust way account deal german music publisher bertelsmann purchase stake report advertising book sale stake loss value', 'dollar gain dollar hit high level euro almost month say trade deficit set highlight government willingness curb spend rise household saving factor help reduce late trading new dollar reach market concern deficit hit greenback recent ahead meeting finance minister send dollar higher early tumble back job think chairman take much sanguine view current account deficit take say take lay set condition current account deficit improve year deficit currency remain peg dollar currency sharp fall recent month therefore make chinese export price highly call shift policy fall deaf recent comment major chinese newspaper time ripe loosening meeting think unlikely produce meaningful movement decision boost interest rate quarter point sixth move many month open differential european enough keep asset look help prop recent fall partly result big budget yawn current account need fund buy bond asset foreign firm announce budget many commentator believe deficit remain close half']\n"
     ]
    }
   ],
   "source": [
    "#lemmatize\n",
    "def lemmatization(texts, allowed_postags=['NOUN','ADJ','VERB','ADV']):\n",
    "    texts_out=[]\n",
    "    for sent in texts:\n",
    "        doc=nlp(' '.join(sent))\n",
    "        texts_out.append(\" \".join([token.lemma_ if token.lemma_ not in ['-PRON-'] else '' for token in doc if token.pos_ in allowed_postags]))\n",
    "    return texts_out\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\", disable=['parser', 'ner'])\n",
    "\n",
    "data_lemmatized = lemmatization(data_words, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])\n",
    "\n",
    "print(data_lemmatized[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer(analyzer='word',       \n",
    "                             min_df=10,                        # minimum read occurences of a word \n",
    "                             stop_words='english',             # remove stop words\n",
    "                             lowercase=True,                   # convert all words to lowercase\n",
    "                             token_pattern='[a-zA-Z0-9]{3,}',  # num chars > 3\n",
    "                             # max_features=50000,             # max number of uniq words\n",
    "                            )\n",
    "\n",
    "data_vectorized = vectorizer.fit_transform(data_lemmatized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LatentDirichletAllocation(learning_decay=0.9, learning_method='online',\n",
      "                          n_components=5, n_jobs=-1, random_state=100)\n"
     ]
    }
   ],
   "source": [
    "lda_model = LatentDirichletAllocation(n_components=5,               # Number of topics\n",
    "                                      max_iter=10,               # Max learning iterations\n",
    "                                      learning_method='online',   \n",
    "                                      random_state=100,          # Random state\n",
    "                                      batch_size=128,            # n docs in each learning iter\n",
    "                                      evaluate_every=-1,       # compute perplexity every n iters, default: Don't\n",
    "                                      n_jobs=-1,               # Use all available CPUs\n",
    "                                      learning_decay=0.9\n",
    "                                     )\n",
    "lda_output = lda_model.fit_transform(data_vectorized)\n",
    "\n",
    "print(lda_model) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log Likelihood:  -1580944.9147908678\n",
      "Perplexity:  1158.9869047231173\n"
     ]
    }
   ],
   "source": [
    "# Log Likelyhood: Higher the better\n",
    "print(\"Log Likelihood: \", lda_model.score(data_vectorized))\n",
    "\n",
    "# Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word)\n",
    "print(\"Perplexity: \", lda_model.perplexity(data_vectorized))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_3ea6d_row0_col0, #T_3ea6d_row0_col1, #T_3ea6d_row1_col3, #T_3ea6d_row1_col5, #T_3ea6d_row2_col2, #T_3ea6d_row2_col4, #T_3ea6d_row2_col5, #T_3ea6d_row3_col0, #T_3ea6d_row3_col3, #T_3ea6d_row4_col1, #T_3ea6d_row4_col2, #T_3ea6d_row4_col3, #T_3ea6d_row4_col5, #T_3ea6d_row5_col0, #T_3ea6d_row5_col3, #T_3ea6d_row6_col0, #T_3ea6d_row6_col3, #T_3ea6d_row7_col2, #T_3ea6d_row7_col3, #T_3ea6d_row7_col5, #T_3ea6d_row8_col0, #T_3ea6d_row8_col2, #T_3ea6d_row9_col2, #T_3ea6d_row9_col5, #T_3ea6d_row10_col0, #T_3ea6d_row11_col2, #T_3ea6d_row11_col3, #T_3ea6d_row11_col5, #T_3ea6d_row12_col1, #T_3ea6d_row12_col5, #T_3ea6d_row13_col0, #T_3ea6d_row13_col1, #T_3ea6d_row13_col2, #T_3ea6d_row13_col5, #T_3ea6d_row14_col2, #T_3ea6d_row14_col5 {\n",
       "  color: green;\n",
       "  font-weight: 700;\n",
       "}\n",
       "#T_3ea6d_row0_col2, #T_3ea6d_row0_col3, #T_3ea6d_row0_col4, #T_3ea6d_row0_col5, #T_3ea6d_row1_col0, #T_3ea6d_row1_col1, #T_3ea6d_row1_col2, #T_3ea6d_row1_col4, #T_3ea6d_row2_col0, #T_3ea6d_row2_col1, #T_3ea6d_row2_col3, #T_3ea6d_row3_col1, #T_3ea6d_row3_col2, #T_3ea6d_row3_col4, #T_3ea6d_row3_col5, #T_3ea6d_row4_col0, #T_3ea6d_row4_col4, #T_3ea6d_row5_col1, #T_3ea6d_row5_col2, #T_3ea6d_row5_col4, #T_3ea6d_row5_col5, #T_3ea6d_row6_col1, #T_3ea6d_row6_col2, #T_3ea6d_row6_col4, #T_3ea6d_row6_col5, #T_3ea6d_row7_col0, #T_3ea6d_row7_col1, #T_3ea6d_row7_col4, #T_3ea6d_row8_col1, #T_3ea6d_row8_col3, #T_3ea6d_row8_col4, #T_3ea6d_row8_col5, #T_3ea6d_row9_col0, #T_3ea6d_row9_col1, #T_3ea6d_row9_col3, #T_3ea6d_row9_col4, #T_3ea6d_row10_col1, #T_3ea6d_row10_col2, #T_3ea6d_row10_col3, #T_3ea6d_row10_col4, #T_3ea6d_row10_col5, #T_3ea6d_row11_col0, #T_3ea6d_row11_col1, #T_3ea6d_row11_col4, #T_3ea6d_row12_col0, #T_3ea6d_row12_col2, #T_3ea6d_row12_col3, #T_3ea6d_row12_col4, #T_3ea6d_row13_col3, #T_3ea6d_row13_col4, #T_3ea6d_row14_col0, #T_3ea6d_row14_col1, #T_3ea6d_row14_col3, #T_3ea6d_row14_col4 {\n",
       "  color: black;\n",
       "  font-weight: 400;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_3ea6d_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >Topic0</th>\n",
       "      <th class=\"col_heading level0 col1\" >Topic1</th>\n",
       "      <th class=\"col_heading level0 col2\" >Topic2</th>\n",
       "      <th class=\"col_heading level0 col3\" >Topic3</th>\n",
       "      <th class=\"col_heading level0 col4\" >Topic4</th>\n",
       "      <th class=\"col_heading level0 col5\" >dominant_topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row0\" class=\"row_heading level0 row0\" >Doc0</th>\n",
       "      <td id=\"T_3ea6d_row0_col0\" class=\"data row0 col0\" >0.570000</td>\n",
       "      <td id=\"T_3ea6d_row0_col1\" class=\"data row0 col1\" >0.290000</td>\n",
       "      <td id=\"T_3ea6d_row0_col2\" class=\"data row0 col2\" >0.090000</td>\n",
       "      <td id=\"T_3ea6d_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row0_col4\" class=\"data row0 col4\" >0.050000</td>\n",
       "      <td id=\"T_3ea6d_row0_col5\" class=\"data row0 col5\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row1\" class=\"row_heading level0 row1\" >Doc1</th>\n",
       "      <td id=\"T_3ea6d_row1_col0\" class=\"data row1 col0\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row1_col1\" class=\"data row1 col1\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row1_col3\" class=\"data row1 col3\" >0.990000</td>\n",
       "      <td id=\"T_3ea6d_row1_col4\" class=\"data row1 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row1_col5\" class=\"data row1 col5\" >3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row2\" class=\"row_heading level0 row2\" >Doc2</th>\n",
       "      <td id=\"T_3ea6d_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row2_col2\" class=\"data row2 col2\" >0.290000</td>\n",
       "      <td id=\"T_3ea6d_row2_col3\" class=\"data row2 col3\" >0.080000</td>\n",
       "      <td id=\"T_3ea6d_row2_col4\" class=\"data row2 col4\" >0.630000</td>\n",
       "      <td id=\"T_3ea6d_row2_col5\" class=\"data row2 col5\" >4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row3\" class=\"row_heading level0 row3\" >Doc3</th>\n",
       "      <td id=\"T_3ea6d_row3_col0\" class=\"data row3 col0\" >0.500000</td>\n",
       "      <td id=\"T_3ea6d_row3_col1\" class=\"data row3 col1\" >0.020000</td>\n",
       "      <td id=\"T_3ea6d_row3_col2\" class=\"data row3 col2\" >0.080000</td>\n",
       "      <td id=\"T_3ea6d_row3_col3\" class=\"data row3 col3\" >0.400000</td>\n",
       "      <td id=\"T_3ea6d_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row3_col5\" class=\"data row3 col5\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row4\" class=\"row_heading level0 row4\" >Doc4</th>\n",
       "      <td id=\"T_3ea6d_row4_col0\" class=\"data row4 col0\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row4_col1\" class=\"data row4 col1\" >0.510000</td>\n",
       "      <td id=\"T_3ea6d_row4_col2\" class=\"data row4 col2\" >0.130000</td>\n",
       "      <td id=\"T_3ea6d_row4_col3\" class=\"data row4 col3\" >0.350000</td>\n",
       "      <td id=\"T_3ea6d_row4_col4\" class=\"data row4 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row4_col5\" class=\"data row4 col5\" >1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row5\" class=\"row_heading level0 row5\" >Doc5</th>\n",
       "      <td id=\"T_3ea6d_row5_col0\" class=\"data row5 col0\" >0.810000</td>\n",
       "      <td id=\"T_3ea6d_row5_col1\" class=\"data row5 col1\" >0.040000</td>\n",
       "      <td id=\"T_3ea6d_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row5_col3\" class=\"data row5 col3\" >0.150000</td>\n",
       "      <td id=\"T_3ea6d_row5_col4\" class=\"data row5 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row5_col5\" class=\"data row5 col5\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row6\" class=\"row_heading level0 row6\" >Doc6</th>\n",
       "      <td id=\"T_3ea6d_row6_col0\" class=\"data row6 col0\" >0.660000</td>\n",
       "      <td id=\"T_3ea6d_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row6_col2\" class=\"data row6 col2\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row6_col3\" class=\"data row6 col3\" >0.330000</td>\n",
       "      <td id=\"T_3ea6d_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row6_col5\" class=\"data row6 col5\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row7\" class=\"row_heading level0 row7\" >Doc7</th>\n",
       "      <td id=\"T_3ea6d_row7_col0\" class=\"data row7 col0\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row7_col1\" class=\"data row7 col1\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row7_col2\" class=\"data row7 col2\" >0.590000</td>\n",
       "      <td id=\"T_3ea6d_row7_col3\" class=\"data row7 col3\" >0.400000</td>\n",
       "      <td id=\"T_3ea6d_row7_col4\" class=\"data row7 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row7_col5\" class=\"data row7 col5\" >2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row8\" class=\"row_heading level0 row8\" >Doc8</th>\n",
       "      <td id=\"T_3ea6d_row8_col0\" class=\"data row8 col0\" >0.560000</td>\n",
       "      <td id=\"T_3ea6d_row8_col1\" class=\"data row8 col1\" >0.090000</td>\n",
       "      <td id=\"T_3ea6d_row8_col2\" class=\"data row8 col2\" >0.350000</td>\n",
       "      <td id=\"T_3ea6d_row8_col3\" class=\"data row8 col3\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row8_col4\" class=\"data row8 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row8_col5\" class=\"data row8 col5\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row9\" class=\"row_heading level0 row9\" >Doc9</th>\n",
       "      <td id=\"T_3ea6d_row9_col0\" class=\"data row9 col0\" >0.090000</td>\n",
       "      <td id=\"T_3ea6d_row9_col1\" class=\"data row9 col1\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row9_col2\" class=\"data row9 col2\" >0.750000</td>\n",
       "      <td id=\"T_3ea6d_row9_col3\" class=\"data row9 col3\" >0.100000</td>\n",
       "      <td id=\"T_3ea6d_row9_col4\" class=\"data row9 col4\" >0.060000</td>\n",
       "      <td id=\"T_3ea6d_row9_col5\" class=\"data row9 col5\" >2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row10\" class=\"row_heading level0 row10\" >Doc10</th>\n",
       "      <td id=\"T_3ea6d_row10_col0\" class=\"data row10 col0\" >0.990000</td>\n",
       "      <td id=\"T_3ea6d_row10_col1\" class=\"data row10 col1\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row10_col2\" class=\"data row10 col2\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row10_col3\" class=\"data row10 col3\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row10_col4\" class=\"data row10 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row10_col5\" class=\"data row10 col5\" >0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row11\" class=\"row_heading level0 row11\" >Doc11</th>\n",
       "      <td id=\"T_3ea6d_row11_col0\" class=\"data row11 col0\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row11_col1\" class=\"data row11 col1\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row11_col2\" class=\"data row11 col2\" >0.410000</td>\n",
       "      <td id=\"T_3ea6d_row11_col3\" class=\"data row11 col3\" >0.580000</td>\n",
       "      <td id=\"T_3ea6d_row11_col4\" class=\"data row11 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row11_col5\" class=\"data row11 col5\" >3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row12\" class=\"row_heading level0 row12\" >Doc12</th>\n",
       "      <td id=\"T_3ea6d_row12_col0\" class=\"data row12 col0\" >0.070000</td>\n",
       "      <td id=\"T_3ea6d_row12_col1\" class=\"data row12 col1\" >0.910000</td>\n",
       "      <td id=\"T_3ea6d_row12_col2\" class=\"data row12 col2\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row12_col3\" class=\"data row12 col3\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row12_col4\" class=\"data row12 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row12_col5\" class=\"data row12 col5\" >1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row13\" class=\"row_heading level0 row13\" >Doc13</th>\n",
       "      <td id=\"T_3ea6d_row13_col0\" class=\"data row13 col0\" >0.200000</td>\n",
       "      <td id=\"T_3ea6d_row13_col1\" class=\"data row13 col1\" >0.130000</td>\n",
       "      <td id=\"T_3ea6d_row13_col2\" class=\"data row13 col2\" >0.660000</td>\n",
       "      <td id=\"T_3ea6d_row13_col3\" class=\"data row13 col3\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row13_col4\" class=\"data row13 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row13_col5\" class=\"data row13 col5\" >2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3ea6d_level0_row14\" class=\"row_heading level0 row14\" >Doc14</th>\n",
       "      <td id=\"T_3ea6d_row14_col0\" class=\"data row14 col0\" >0.080000</td>\n",
       "      <td id=\"T_3ea6d_row14_col1\" class=\"data row14 col1\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row14_col2\" class=\"data row14 col2\" >0.920000</td>\n",
       "      <td id=\"T_3ea6d_row14_col3\" class=\"data row14 col3\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row14_col4\" class=\"data row14 col4\" >0.000000</td>\n",
       "      <td id=\"T_3ea6d_row14_col5\" class=\"data row14 col5\" >2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1c87f4bc310>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topicnames = [\"Topic\" + str(i) for i in range(lda_model.n_components)]\n",
    "\n",
    "# index names\n",
    "docnames = [\"Doc\" + str(i) for i in range(len(data))]\n",
    "\n",
    "# Make the pandas dataframe\n",
    "df_document_topic = pd.DataFrame(np.round(lda_output, 2), columns=topicnames, index=docnames)\n",
    "\n",
    "# Get dominant topic for each document\n",
    "dominant_topic = np.argmax(df_document_topic.values, axis=1)\n",
    "df_document_topic['dominant_topic'] = dominant_topic\n",
    "\n",
    "# Styling\n",
    "def color_green(val):\n",
    "    color = 'green' if val > .1 else 'black'\n",
    "    return 'color: {col}'.format(col=color)\n",
    "\n",
    "def make_bold(val):\n",
    "    weight = 700 if val > .1 else 400\n",
    "    return 'font-weight: {weight}'.format(weight=weight)\n",
    "\n",
    "# Apply Style\n",
    "df_document_topics = df_document_topic.head(15).style.applymap(color_green).applymap(make_bold)\n",
    "df_document_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pyLDAvis' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-22-d5bfafec2137>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpyLDAvis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menable_notebook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mpanel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpyLDAvis\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprepare\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlda_model\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata_vectorized\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvectorizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmds\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'tsne'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mpanel\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'pyLDAvis' is not defined"
     ]
    }
   ],
   "source": [
    "pyLDAvis.enable_notebook()\n",
    "panel = pyLDAvis.sklearn.prepare(lda_model, data_vectorized, vectorizer, mds='tsne')\n",
    "panel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.11 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.11"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d9f8fdbca0c29d8b6577bd292967f8689a2652650c8a2af83505eef18611a225"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}