0500_READING_NEW_PAPERS_Gensim.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.read_csv('data/articles1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>publication</th>\n",
       "      <th>author</th>\n",
       "      <th>date</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>url</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>53293</td>\n",
       "      <td>73471</td>\n",
       "      <td>Patriots Day Is Best When It Digs Past the Heroism</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>David Sims</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Patriots Day, Peter Berg’s new thriller that recreates the 2013 Boston Marathon bombing and the ensuing manhunt that followed it, is a surprisingly oblique, morally ambiguous movie from a typically straightforward filmmaker. Patriots Day takes an unexpectedly cynical view of the chaos, rash   an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>53294</td>\n",
       "      <td>73472</td>\n",
       "      <td>A Break in the Search for the Origin of Complex Life</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Ed Yong</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>In Norse mythology, humans and our world were created by a pantheon of gods who lived in the realm of Asgard. As it turns out, these stories have a grain of truth to them. Thanks to a team of scientists led by Thijs Ettema, Asgard is now also the name of a large clan of microbes. Its members, wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>53295</td>\n",
       "      <td>73474</td>\n",
       "      <td>Obama’s Ingenious Mention of Atticus Finch</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Spencer Kornhaber</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>“If our democracy is to work in this increasingly diverse nation,” Barack Obama said in his farewell address last night, “each one of us must try to heed the advice of one of the great characters in American fiction, Atticus Finch. ” He then quoted Finch: “You never really understand a person un...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53296</td>\n",
       "      <td>73475</td>\n",
       "      <td>Donald Trump Meets, and Assails, the Press</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>David A. Graham</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Updated on January 11 at 5:05 p. m. In his first press conference since July 2016,   Donald Trump took only a few questions but made news on several fronts, saying he accepted the conclusion that Russia conducted hacks on top Democrats, bashing the press, and refusing once again to release his t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>53297</td>\n",
       "      <td>73476</td>\n",
       "      <td>Trump: ’I Think’ Hacking Was Russian</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Kaveh Waddell</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Updated at 12:25 p. m. After months of equivocating on the origin of cyberattacks that targeted Democrats before the election,   Donald Trump said Wednesday that he thinks Russia was behind the intrusions. “As for hacking, I think it was Russian,” Trump said at a press conference in New York. “B...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0     id                                                 title  \\\n",
       "0       53293  73471    Patriots Day Is Best When It Digs Past the Heroism   \n",
       "1       53294  73472  A Break in the Search for the Origin of Complex Life   \n",
       "2       53295  73474            Obama’s Ingenious Mention of Atticus Finch   \n",
       "3       53296  73475            Donald Trump Meets, and Assails, the Press   \n",
       "4       53297  73476                  Trump: ’I Think’ Hacking Was Russian   \n",
       "\n",
       "  publication             author        date    year  month  url  \\\n",
       "0    Atlantic         David Sims  2017-01-11  2017.0    1.0  NaN   \n",
       "1    Atlantic            Ed Yong  2017-01-11  2017.0    1.0  NaN   \n",
       "2    Atlantic  Spencer Kornhaber  2017-01-11  2017.0    1.0  NaN   \n",
       "3    Atlantic    David A. Graham  2017-01-11  2017.0    1.0  NaN   \n",
       "4    Atlantic      Kaveh Waddell  2017-01-11  2017.0    1.0  NaN   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                       content  \n",
       "0  Patriots Day, Peter Berg’s new thriller that recreates the 2013 Boston Marathon bombing and the ensuing manhunt that followed it, is a surprisingly oblique, morally ambiguous movie from a typically straightforward filmmaker. Patriots Day takes an unexpectedly cynical view of the chaos, rash   an...  \n",
       "1  In Norse mythology, humans and our world were created by a pantheon of gods who lived in the realm of Asgard. As it turns out, these stories have a grain of truth to them. Thanks to a team of scientists led by Thijs Ettema, Asgard is now also the name of a large clan of microbes. Its members, wh...  \n",
       "2  “If our democracy is to work in this increasingly diverse nation,” Barack Obama said in his farewell address last night, “each one of us must try to heed the advice of one of the great characters in American fiction, Atticus Finch. ” He then quoted Finch: “You never really understand a person un...  \n",
       "3  Updated on January 11 at 5:05 p. m. In his first press conference since July 2016,   Donald Trump took only a few questions but made news on several fronts, saying he accepted the conclusion that Russia conducted hacks on top Democrats, bashing the press, and refusing once again to release his t...  \n",
       "4  Updated at 12:25 p. m. After months of equivocating on the origin of cyberattacks that targeted Democrats before the election,   Donald Trump said Wednesday that he thinks Russia was behind the intrusions. “As for hacking, I think it was Russian,” Trump said at a press conference in New York. “B...  "
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2 = pd.read_csv('data/articles2.csv')\n",
    "df_2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### let's select the first 50 new papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_titles = df_2['title'][:50].array\n",
    "new_papers = df_2['content'][:50].array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Trump Thanks L.L. Bean, the Latest Retailer Caught Between Him and His Critics'"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_titles[34]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import nltk\n",
    "import gensim\n",
    "\n",
    "dictionary = gensim.corpora.Dictionary.load('models/dictionary.gensim')\n",
    "\n",
    "with open(\"lists/bow_corpus.txt\", \"rb\") as fp:   # Unpickling\n",
    "    bow_corpus = pickle.load(fp)\n",
    "\n",
    "with open(\"lists/norm_corpus_bigrams.txt\", \"rb\") as fp: \n",
    "    norm_corpus_bigrams = pickle.load(fp)\n",
    "\n",
    "with open(\"lists/norm_papers.txt\", \"rb\") as fp:\n",
    "    norm_papers = pickle.load(fp)\n",
    "\n",
    "with open(\"lists/pre_papers.txt\", \"rb\") as fp:   \n",
    "    pre_papers = pickle.load(fp)\n",
    "\n",
    "with open(\"lists/pre_titles.txt\", \"rb\") as fp:  \n",
    "    pre_titles = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PREPROCESS NEW PAPERS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "first preprcoess these new papers and extract features using the same sequence of steps we followed when building the topic models.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 810 µs, sys: 1.42 ms, total: 2.23 ms\n",
      "Wall time: 2.2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import nltk\n",
    "\n",
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "wtk = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "wnl = nltk.stem.wordnet.WordNetLemmatizer()\n",
    "\n",
    "def normalise_corpus(papers, titles):\n",
    "    norm_papers = []\n",
    "    pre_papers = []\n",
    "    pre_titles = []\n",
    "    for i in range(len(papers)):\n",
    "        paper = papers[i]\n",
    "        title = titles[i]\n",
    "\n",
    "        paper = paper.lower()\n",
    "        paper_tokens = [token.strip() for token in wtk.tokenize(paper)]\n",
    "        paper_tokens = [wnl.lemmatize(token) for token in paper_tokens if not token.isnumeric()]\n",
    "        paper_tokens = [token for token in paper_tokens if len(token) > 1]\n",
    "        paper_tokens = [token for token in paper_tokens if token not in stop_words]\n",
    "        paper_tokens = list(filter(None, paper_tokens))\n",
    "\n",
    "        if paper_tokens:\n",
    "            norm_papers.append(paper_tokens)\n",
    "            pre_papers.append(paper)\n",
    "            pre_titles.append(title)\n",
    "\n",
    "    return norm_papers, pre_papers, pre_titles\n",
    "\n",
    "# we have pre_papers and pre_titles because the normalizing function removes empty papers and titles\n",
    "# so for consistency the papers and titles that we perform LDA on will be kept "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### let's create a text wrangling and feature engineering pipeline, which should match the same steps we followed when training our topic model.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_model = gensim.models.phrases.Phraser.load('models/bigram_model.gensim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_preprocessing_pipeline(documents, normaliser_fn, bigram_model, titles):\n",
    "    norm_docs, pre_papers, pre_titles = normaliser_fn(documents, titles)\n",
    "    norm_docs_bigrams = bigram_model[norm_docs]\n",
    "    return norm_docs_bigrams, pre_papers, pre_titles\n",
    "\n",
    "def bow_features_pipeline(tokenized_docs, dictionary):\n",
    "    paper_bow_features = [dictionary.doc2bow(text)\n",
    "                              for text in tokenized_docs]\n",
    "    return paper_bow_features\n",
    "\n",
    "norm_new_papers, new_pre_papers, new_pre_titles = text_preprocessing_pipeline(documents=new_papers,\n",
    "                                                                    normaliser_fn=normalise_corpus,\n",
    "                                                                    bigram_model=bigram_model, \n",
    "                                                                    titles=new_titles)\n",
    "\n",
    "norm_bow_features = bow_features_pipeline(tokenized_docs=norm_new_papers,\n",
    "                                         dictionary=dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['patriot', 'day', 'peter', 'berg', 'new', 'thriller', 'recreates', 'boston_marathon', 'bombing', 'ensuing', 'manhunt', 'followed', 'surprisingly', 'oblique', 'morally', 'ambiguous', 'movie', 'typically', 'straightforward', 'filmmaker', 'patriot', 'day', 'take', 'unexpectedly', 'cynical', 'view', 'chaos', 'rash', 'bureaucratic', 'infighting']\n"
     ]
    }
   ],
   "source": [
    "print(norm_new_papers[0][:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(22, 1), (26, 1), (31, 3), (36, 2), (39, 2), (42, 1), (57, 1), (69, 1), (85, 2), (96, 1), (97, 1), (108, 1), (140, 1), (144, 1), (149, 1), (157, 1), (167, 1), (168, 2), (174, 2), (177, 2), (183, 1), (230, 1), (236, 1), (237, 1), (240, 1), (253, 1), (261, 2), (266, 2), (280, 1), (287, 1)]\n"
     ]
    }
   ],
   "source": [
    "print(norm_bow_features[0][:30])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LOAD LDA MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOPICS = 25\n",
    "\n",
    "load_lda_model = gensim.models.ldamodel.LdaModel.load('models/gensim/model_'+str(TOPICS)+'.gensim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Terms per Topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Topic1</th>\n",
       "      <td>people, even, american, many, right, would, world, political, america, like, way, country, also, issue, power, time, new, make, fact, history</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic2</th>\n",
       "      <td>isi, military, syria, russia, government, iran, country, force, war, russian, group, united_state, turkey, iraq, also, putin, syrian, official, leader, obama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic3</th>\n",
       "      <td>book, cuban, abortion, cuba, king, life, castro, india, lewis, lee, indian, canada, death, canadian, woman, planned_parenthood, jackson, hurricane, baby, american</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic4</th>\n",
       "      <td>official, investigation, email, would, russia, president, fbi, russian, report, information, white_house, former, government, told_cnn, committee, intelligence, administration, comey, also, statement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic5</th>\n",
       "      <td>company, business, million, apple, tesla, new, year, market, also, product, google, sale, investor, billion, money, customer, according, employee, car, technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic6</th>\n",
       "      <td>trump, clinton, president, donald_trump, campaign, obama, hillary_clinton, white_house, election, former, mr, speech, also, candidate, say, first, would, american, country, day</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic7</th>\n",
       "      <td>facebook, news, twitter, video, medium, story, show, cnn, time, social_medium, post, tweet, user, online, also, report, website, network, ad, site</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic8</th>\n",
       "      <td>attack, muslim, people, killed, isi, terrorist, airline, security, airport, terrorism, two, bomb, attacker, bombing, according, pakistan, group, paris, threat, terrorist_attack</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic9</th>\n",
       "      <td>car, water, could, space, area, plane, fire, flight, cnn, storm, vehicle, home, foot, air, hour, first, system, passenger, driver, road</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic10</th>\n",
       "      <td>game, team, first, player, sport, win, season, two, year, final, second, play, time, world, three, fan, point, football, ball, back</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic11</th>\n",
       "      <td>country, immigration, mexico, refugee, immigrant, border, united_state, migrant, mexican, order, executive_order, security, wall, agent, government, ban, citizen, people, visa, policy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic12</th>\n",
       "      <td>study, drug, health, patient, also, case, research, may, found, doctor, food, researcher, year, people, according, medical, use, risk, dr, data</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic13</th>\n",
       "      <td>china, north_korea, chinese, country, official, japan, sea, international, nuclear, military, would, missile, island, united_state, two, aircraft, un, philippine, according, ship</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic14</th>\n",
       "      <td>would, year, american, plan, job, economy, policy, percent, cost, government, tax, new, million, people, could, also, worker, trade, economic, program</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic15</th>\n",
       "      <td>say, people, like, get, think, going, know, would, thing, want, time, way, go, really, see, lot, make, back, could, look</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic16</th>\n",
       "      <td>student, school, university, college, state, education, campus, teacher, kasich, program, class, community, high_school, public, parent, child, kid, law, professor, board</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic17</th>\n",
       "      <td>march, day, california, july, june, april, cnn, may, time, september, january, uk, according, february, brown, november, los_angeles, october, december, august</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic18</th>\n",
       "      <td>woman, black, white, men, female, music, song, sex, also, gay, girl, young, year, show, man, love, color, race, transgender, singer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic19</th>\n",
       "      <td>family, child, life, father, mother, home, son, told, parent, day, church, friend, death, daughter, girl, died, two, wife, boy, brother</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic20</th>\n",
       "      <td>police, officer, case, court, gun, two, according, prison, authority, man, charge, judge, shooting, state, crime, victim, suspect, prosecutor, law, arrested</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic21</th>\n",
       "      <td>europe, party, britain, european, country, france, british, germany, french, london, german, government, prime_minister, leader, may, vote, election, mr, remain, brexit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic22</th>\n",
       "      <td>year, new_york, work, time, first, mr, city, new, home, building, two, world, project, name, life, art, known, became, hotel, also</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic23</th>\n",
       "      <td>film, show, movie, star, actor, character, eu, hollywood, series, best, season, tv, award, episode, role, producer, director, television, audience, netflix</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic24</th>\n",
       "      <td>republican, state, vote, democrat, party, voter, election, would, candidate, cruz, support, poll, senate, bill, gop, conservative, house, sander, democratic, rubio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic25</th>\n",
       "      <td>city, police, people, protest, street, protester, israel, official, area, local, mayor, group, officer, resident, saturday, police_officer, violence, outside, state, incident</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                 Terms per Topic\n",
       "Topic1   people, even, american, many, right, would, world, political, america, like, way, country, also, issue, power, time, new, make, fact, history                                                          \n",
       "Topic2   isi, military, syria, russia, government, iran, country, force, war, russian, group, united_state, turkey, iraq, also, putin, syrian, official, leader, obama                                          \n",
       "Topic3   book, cuban, abortion, cuba, king, life, castro, india, lewis, lee, indian, canada, death, canadian, woman, planned_parenthood, jackson, hurricane, baby, american                                     \n",
       "Topic4   official, investigation, email, would, russia, president, fbi, russian, report, information, white_house, former, government, told_cnn, committee, intelligence, administration, comey, also, statement\n",
       "Topic5   company, business, million, apple, tesla, new, year, market, also, product, google, sale, investor, billion, money, customer, according, employee, car, technology                                     \n",
       "Topic6   trump, clinton, president, donald_trump, campaign, obama, hillary_clinton, white_house, election, former, mr, speech, also, candidate, say, first, would, american, country, day                       \n",
       "Topic7   facebook, news, twitter, video, medium, story, show, cnn, time, social_medium, post, tweet, user, online, also, report, website, network, ad, site                                                     \n",
       "Topic8   attack, muslim, people, killed, isi, terrorist, airline, security, airport, terrorism, two, bomb, attacker, bombing, according, pakistan, group, paris, threat, terrorist_attack                       \n",
       "Topic9   car, water, could, space, area, plane, fire, flight, cnn, storm, vehicle, home, foot, air, hour, first, system, passenger, driver, road                                                                \n",
       "Topic10  game, team, first, player, sport, win, season, two, year, final, second, play, time, world, three, fan, point, football, ball, back                                                                    \n",
       "Topic11  country, immigration, mexico, refugee, immigrant, border, united_state, migrant, mexican, order, executive_order, security, wall, agent, government, ban, citizen, people, visa, policy                \n",
       "Topic12  study, drug, health, patient, also, case, research, may, found, doctor, food, researcher, year, people, according, medical, use, risk, dr, data                                                        \n",
       "Topic13  china, north_korea, chinese, country, official, japan, sea, international, nuclear, military, would, missile, island, united_state, two, aircraft, un, philippine, according, ship                     \n",
       "Topic14  would, year, american, plan, job, economy, policy, percent, cost, government, tax, new, million, people, could, also, worker, trade, economic, program                                                 \n",
       "Topic15  say, people, like, get, think, going, know, would, thing, want, time, way, go, really, see, lot, make, back, could, look                                                                               \n",
       "Topic16  student, school, university, college, state, education, campus, teacher, kasich, program, class, community, high_school, public, parent, child, kid, law, professor, board                             \n",
       "Topic17  march, day, california, july, june, april, cnn, may, time, september, january, uk, according, february, brown, november, los_angeles, october, december, august                                        \n",
       "Topic18  woman, black, white, men, female, music, song, sex, also, gay, girl, young, year, show, man, love, color, race, transgender, singer                                                                    \n",
       "Topic19  family, child, life, father, mother, home, son, told, parent, day, church, friend, death, daughter, girl, died, two, wife, boy, brother                                                                \n",
       "Topic20  police, officer, case, court, gun, two, according, prison, authority, man, charge, judge, shooting, state, crime, victim, suspect, prosecutor, law, arrested                                           \n",
       "Topic21  europe, party, britain, european, country, france, british, germany, french, london, german, government, prime_minister, leader, may, vote, election, mr, remain, brexit                               \n",
       "Topic22  year, new_york, work, time, first, mr, city, new, home, building, two, world, project, name, life, art, known, became, hotel, also                                                                     \n",
       "Topic23  film, show, movie, star, actor, character, eu, hollywood, series, best, season, tv, award, episode, role, producer, director, television, audience, netflix                                            \n",
       "Topic24  republican, state, vote, democrat, party, voter, election, would, candidate, cruz, support, poll, senate, bill, gop, conservative, house, sander, democratic, rubio                                    \n",
       "Topic25  city, police, people, protest, street, protester, israel, official, area, local, mayor, group, officer, resident, saturday, police_officer, violence, outside, state, incident                         "
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics = [[(term, round(wt, 3))\n",
    "               for term, wt in load_lda_model.show_topic(n, topn=20)]\n",
    "                   for n in range(0, load_lda_model.num_topics)]\n",
    "\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "topics_df = pd.DataFrame([', '.join([term for term, wt in topic])\n",
    "                              for topic in topics],\n",
    "                         columns = ['Terms per Topic'],\n",
    "                         index=['Topic'+str(t) for t in range(1, load_lda_model.num_topics+1)]\n",
    "                         )\n",
    "topics_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PREDICT NEW TOPICS OF PAPERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_topic_predictions(topic_model, corpus, topn=3):\n",
    "    topic_predictions = topic_model[corpus]\n",
    "    best_topics = [[(topic, round(wt, 3)) \n",
    "                        for topic, wt in sorted(topic_predictions[i], \n",
    "                                                key=lambda row: -row[1])[:topn]] \n",
    "                            for i in range(len(topic_predictions))]\n",
    "    return best_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_preds = get_topic_predictions(topic_model=load_lda_model, \n",
    "                                    corpus=norm_bow_features, topn=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### building a results df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.DataFrame()\n",
    "results_df['Papers'] = range(1, len(new_pre_papers)+1)\n",
    "results_df['Dominant Topics'] = [[topic_num+1 for topic_num, wt in item] for item in topic_preds]\n",
    "res = results_df.set_index(['Papers'])['Dominant Topics'].apply(pd.Series).stack().reset_index(level=1, drop=True)\n",
    "results_df = pd.DataFrame({'Dominant Topics': res.values}, index=res.index)\n",
    "results_df['Contribution %'] = [topic_wt for topic_list in \n",
    "                                        [[round(wt*100, 2) \n",
    "                                              for topic_num, wt in item] \n",
    "                                                 for item in topic_preds] \n",
    "                                    for topic_wt in topic_list]\n",
    "\n",
    "results_df['Topic Desc'] = [topics_df.iloc[t-1]['Terms per Topic'] for t in results_df['Dominant Topics'].values]\n",
    "results_df['Title'] = [new_pre_titles[i-1][:200] for i in results_df.index.values]\n",
    "results_df['Paper Desc'] = [new_pre_papers[i-1][:200] for i in results_df.index.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dominant Topics</th>\n",
       "      <th>Contribution %</th>\n",
       "      <th>Topic Desc</th>\n",
       "      <th>Title</th>\n",
       "      <th>Paper Desc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Papers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4</td>\n",
       "      <td>67.4</td>\n",
       "      <td>official, investigation, email, would, russia, president, fbi, russian, report, information, white_house, former, government, told_cnn, committee, intelligence, administration, comey, also, statement</td>\n",
       "      <td>Trump: ’I Think’ Hacking Was Russian</td>\n",
       "      <td>updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>12</td>\n",
       "      <td>55.0</td>\n",
       "      <td>study, drug, health, patient, also, case, research, may, found, doctor, food, researcher, year, people, according, medical, use, risk, dr, data</td>\n",
       "      <td>Could Cancer Drugs Treat Autism?</td>\n",
       "      <td>five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>50.8</td>\n",
       "      <td>people, even, american, many, right, would, world, political, america, like, way, country, also, issue, power, time, new, make, fact, history</td>\n",
       "      <td>Obama’s Ingenious Mention of Atticus Finch</td>\n",
       "      <td>“if our democracy is to work in this increasingly diverse nation,” barack obama said in his farewell address last night, “each one of us must try to heed the advice of one of the great characters in a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>14</td>\n",
       "      <td>50.6</td>\n",
       "      <td>would, year, american, plan, job, economy, policy, percent, cost, government, tax, new, million, people, could, also, worker, trade, economic, program</td>\n",
       "      <td>Obama Frames His Economic Legacy</td>\n",
       "      <td>in barack obama’s final speech as president, he touted nearly a decade of economic improvement. “if i had told you eight years ago that america would reverse a great recession, reboot our auto industr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>24</td>\n",
       "      <td>46.6</td>\n",
       "      <td>republican, state, vote, democrat, party, voter, election, would, candidate, cruz, support, poll, senate, bill, gop, conservative, house, sander, democratic, rubio</td>\n",
       "      <td>Obamacare Repeal Moves a Step Closer to Reality</td>\n",
       "      <td>republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>13.3</td>\n",
       "      <td>trump, clinton, president, donald_trump, campaign, obama, hillary_clinton, white_house, election, former, mr, speech, also, candidate, say, first, would, american, country, day</td>\n",
       "      <td>Trump: ’I Think’ Hacking Was Russian</td>\n",
       "      <td>updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>15</td>\n",
       "      <td>13.2</td>\n",
       "      <td>say, people, like, get, think, going, know, would, thing, want, time, way, go, really, see, lot, make, back, could, look</td>\n",
       "      <td>The Atlantic Daily: Loose Ends and Legacy</td>\n",
       "      <td>this article is part of a feature we also send out via email as the atlantic daily, a newsletter with stories, ideas, and images from the atlantic, written specially for subscribers. to sign u</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>5</td>\n",
       "      <td>12.8</td>\n",
       "      <td>company, business, million, apple, tesla, new, year, market, also, product, google, sale, investor, billion, money, customer, according, employee, car, technology</td>\n",
       "      <td>The Revolt of Working Parents</td>\n",
       "      <td>“there is no way you can be a good mother while achieving what i aspire.” “let’s face it. it’s a man’s world. the woman always stays home with the child. ” “it’s hard to do this job with two kids. ” t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>18</td>\n",
       "      <td>12.1</td>\n",
       "      <td>woman, black, white, men, female, music, song, sex, also, gay, girl, young, year, show, man, love, color, race, transgender, singer</td>\n",
       "      <td>How Victoria Aims to Connect With Young Women</td>\n",
       "      <td>victoria, a new miniseries charting the famous queen’s early reign, premieres sunday on pbs’s masterpiece —  nearly 180 years after the monarch ascended the british throne and five days before donald</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>21</td>\n",
       "      <td>11.9</td>\n",
       "      <td>europe, party, britain, european, country, france, british, germany, french, london, german, government, prime_minister, leader, may, vote, election, mr, remain, brexit</td>\n",
       "      <td>How Swimming Classes Became an Integration Issue in Switzerland</td>\n",
       "      <td>the european court of human rights (echr) ruled unanimously tuesday in favor of a swiss school’s decision not to exempt two muslim students from its obligatory   swimming lessons, finding that the sch</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Dominant Topics  Contribution %  \\\n",
       "Papers                                    \n",
       "5                     4            67.4   \n",
       "28                   12            55.0   \n",
       "3                     1            50.8   \n",
       "7                    14            50.6   \n",
       "45                   24            46.6   \n",
       "...                 ...             ...   \n",
       "5                     6            13.3   \n",
       "10                   15            13.2   \n",
       "32                    5            12.8   \n",
       "46                   18            12.1   \n",
       "29                   21            11.9   \n",
       "\n",
       "                                                                                                                                                                                                     Topic Desc  \\\n",
       "Papers                                                                                                                                                                                                            \n",
       "5       official, investigation, email, would, russia, president, fbi, russian, report, information, white_house, former, government, told_cnn, committee, intelligence, administration, comey, also, statement   \n",
       "28                                                              study, drug, health, patient, also, case, research, may, found, doctor, food, researcher, year, people, according, medical, use, risk, dr, data   \n",
       "3                                                                 people, even, american, many, right, would, world, political, america, like, way, country, also, issue, power, time, new, make, fact, history   \n",
       "7                                                        would, year, american, plan, job, economy, policy, percent, cost, government, tax, new, million, people, could, also, worker, trade, economic, program   \n",
       "45                                          republican, state, vote, democrat, party, voter, election, would, candidate, cruz, support, poll, senate, bill, gop, conservative, house, sander, democratic, rubio   \n",
       "...                                                                                                                                                                                                         ...   \n",
       "5                              trump, clinton, president, donald_trump, campaign, obama, hillary_clinton, white_house, election, former, mr, speech, also, candidate, say, first, would, american, country, day   \n",
       "10                                                                                     say, people, like, get, think, going, know, would, thing, want, time, way, go, really, see, lot, make, back, could, look   \n",
       "32                                           company, business, million, apple, tesla, new, year, market, also, product, google, sale, investor, billion, money, customer, according, employee, car, technology   \n",
       "46                                                                          woman, black, white, men, female, music, song, sex, also, gay, girl, young, year, show, man, love, color, race, transgender, singer   \n",
       "29                                     europe, party, britain, european, country, france, british, germany, french, london, german, government, prime_minister, leader, may, vote, election, mr, remain, brexit   \n",
       "\n",
       "                                                                  Title  \\\n",
       "Papers                                                                    \n",
       "5                                  Trump: ’I Think’ Hacking Was Russian   \n",
       "28                                     Could Cancer Drugs Treat Autism?   \n",
       "3                            Obama’s Ingenious Mention of Atticus Finch   \n",
       "7                                      Obama Frames His Economic Legacy   \n",
       "45                      Obamacare Repeal Moves a Step Closer to Reality   \n",
       "...                                                                 ...   \n",
       "5                                  Trump: ’I Think’ Hacking Was Russian   \n",
       "10                           The Atlantic Daily: Loose Ends and Legacy    \n",
       "32                                        The Revolt of Working Parents   \n",
       "46                       How Victoria Aims to Connect With Young Women    \n",
       "29      How Swimming Classes Became an Integration Issue in Switzerland   \n",
       "\n",
       "                                                                                                                                                                                                      Paper Desc  \n",
       "Papers                                                                                                                                                                                                            \n",
       "5       updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus  \n",
       "28      five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau  \n",
       "3       “if our democracy is to work in this increasingly diverse nation,” barack obama said in his farewell address last night, “each one of us must try to heed the advice of one of the great characters in a  \n",
       "7       in barack obama’s final speech as president, he touted nearly a decade of economic improvement. “if i had told you eight years ago that america would reverse a great recession, reboot our auto industr  \n",
       "45      republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i  \n",
       "...                                                                                                                                                                                                          ...  \n",
       "5       updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus  \n",
       "10              this article is part of a feature we also send out via email as the atlantic daily, a newsletter with stories, ideas, and images from the atlantic, written specially for subscribers. to sign u  \n",
       "32      “there is no way you can be a good mother while achieving what i aspire.” “let’s face it. it’s a man’s world. the woman always stays home with the child. ” “it’s hard to do this job with two kids. ” t  \n",
       "46      victoria, a new miniseries charting the famous queen’s early reign, premieres sunday on pbs’s masterpiece —  nearly 180 years after the monarch ascended the british throne and five days before donald   \n",
       "29      the european court of human rights (echr) ruled unanimously tuesday in favor of a swiss school’s decision not to exempt two muslim students from its obligatory   swimming lessons, finding that the sch  \n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', 300)\n",
    "\n",
    "results_df.sort_values(by='Contribution %', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looking at the generated topics for the new, previously unseen papers, I would say our model has done an excellent job!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PREDICTING WITH MALLET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<gensim.models.ldamodel.LdaModel at 0x7fa5506eac50>"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_lda_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOPICS = 25\n",
    "\n",
    "load_lda_model = gensim.models.wrappers.LdaMallet.load('models/mallet/model_'+str(TOPICS)+'.gensim')\n",
    "\n",
    "\n",
    "# convert the ldaMallet to LdaModel. It was the only way to get some result with loading mallet model.\n",
    "load_lda_model = gensim.models.wrappers.ldamallet.malletmodel2ldamodel(load_lda_model)\n",
    "\n",
    "topics = [[(term, round(wt, 3))\n",
    "               for term, wt in load_lda_model.show_topic(n, topn=20)]\n",
    "                   for n in range(0, load_lda_model.num_topics)]\n",
    "\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "\n",
    "topics_df = pd.DataFrame([', '.join([term for term, wt in topic])\n",
    "                              for topic in topics],\n",
    "                         columns = ['Terms per Topic'],\n",
    "                         index=['Topic'+str(t) for t in range(1, load_lda_model.num_topics+1)]\n",
    "                         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_topic_predictions(topic_model, corpus, topn=3):\n",
    "    topic_predictions = topic_model[corpus]\n",
    "    best_topics = [[(topic, round(wt, 3)) \n",
    "                        for topic, wt in sorted(topic_predictions[i], \n",
    "                                                key=lambda row: -row[1])[:topn]] \n",
    "                            for i in range(len(topic_predictions))]\n",
    "    return best_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_preds = get_topic_predictions(topic_model=load_lda_model, \n",
    "                                    corpus=norm_bow_features, topn=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "lines_to_next_cell": 0
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dominant Topics</th>\n",
       "      <th>Contribution %</th>\n",
       "      <th>Topic Desc</th>\n",
       "      <th>Title</th>\n",
       "      <th>Paper Desc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Papers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>13</td>\n",
       "      <td>67.4</td>\n",
       "      <td>study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result</td>\n",
       "      <td>Could Cancer Drugs Treat Autism?</td>\n",
       "      <td>five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>12</td>\n",
       "      <td>58.8</td>\n",
       "      <td>republican, bill, democrat, house, senate, congress, ryan, senator, vote, plan, session, conservative, member, committee, people, health_care, american, obamacare, support, president</td>\n",
       "      <td>Obamacare Repeal Moves a Step Closer to Reality</td>\n",
       "      <td>republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>13</td>\n",
       "      <td>52.7</td>\n",
       "      <td>study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result</td>\n",
       "      <td>Resistance to the Antibiotic of Last Resort Is Silently Spreading</td>\n",
       "      <td>the alarm bells sounded on november 18, 2015. antibiotic resistance is usually a   crisis, one of the reasons its danger can be hard to convey. one by one, over the years, the drugs used to fight the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>16</td>\n",
       "      <td>51.1</td>\n",
       "      <td>show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story</td>\n",
       "      <td>The Atlantic’s Week in Culture</td>\n",
       "      <td>don’t miss, la la land’s   nostalgia —  david sims analyzes the nature of time in damien chazelle’s film, which pays homage to old hollywood while portraying the limits of worshipping the past. filmpa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>47.2</td>\n",
       "      <td>official, email, investigation, report, clinton, information, russian, fbi, intelligence, russia, campaign, comey, document, committee, evidence, government, source, time, agency, hillary_clinton</td>\n",
       "      <td>Trump: ’I Think’ Hacking Was Russian</td>\n",
       "      <td>updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>14</td>\n",
       "      <td>7.3</td>\n",
       "      <td>people, medium, thing, news, time, story, breitbart_news, twitter, show, question, interview, lot, fact, breitbart, fox_news, asked, talk, reporter, tweet, guy</td>\n",
       "      <td>Is Obamacare Just Bad Branding?</td>\n",
       "      <td>the senate began the process of dismantling obamacare early thursday morning, passing a budget blueprint that sets the stage for a formal repeal vote, potentially within a few weeks. the step is, for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7</td>\n",
       "      <td>6.8</td>\n",
       "      <td>trump, president, obama, white_house, donald_trump, administration, day, american, washington, meeting, office, policy, president_barack, national_security, presidency, friday, issue, election, told, statement</td>\n",
       "      <td>Trump: ’I Think’ Hacking Was Russian</td>\n",
       "      <td>updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>24</td>\n",
       "      <td>6.8</td>\n",
       "      <td>people, black, america, american, white, history, church, world, christian, community, political, word, movement, nation, speech, country, life, group, gay, god</td>\n",
       "      <td>The Atlantic’s Week in Culture</td>\n",
       "      <td>don’t miss, la la land’s   nostalgia —  david sims analyzes the nature of time in damien chazelle’s film, which pays homage to old hollywood while portraying the limits of worshipping the past. filmpa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>19</td>\n",
       "      <td>5.7</td>\n",
       "      <td>american, job, people, worker, country, work, tax, year, policy, economy, economic, make, trade, america, good, plan, world, government, system, problem</td>\n",
       "      <td>Obamacare Repeal Moves a Step Closer to Reality</td>\n",
       "      <td>republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>15</td>\n",
       "      <td>5.4</td>\n",
       "      <td>time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small</td>\n",
       "      <td>Could Cancer Drugs Treat Autism?</td>\n",
       "      <td>five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Dominant Topics  Contribution %  \\\n",
       "Papers                                    \n",
       "28                   13            67.4   \n",
       "45                   12            58.8   \n",
       "30                   13            52.7   \n",
       "38                   16            51.1   \n",
       "5                     2            47.2   \n",
       "...                 ...             ...   \n",
       "26                   14             7.3   \n",
       "5                     7             6.8   \n",
       "38                   24             6.8   \n",
       "45                   19             5.7   \n",
       "28                   15             5.4   \n",
       "\n",
       "                                                                                                                                                                                                               Topic Desc  \\\n",
       "Papers                                                                                                                                                                                                                      \n",
       "28                                                             study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result   \n",
       "45                                 republican, bill, democrat, house, senate, congress, ryan, senator, vote, plan, session, conservative, member, committee, people, health_care, american, obamacare, support, president   \n",
       "30                                                             study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result   \n",
       "38                                                                      show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story   \n",
       "5                     official, email, investigation, report, clinton, information, russian, fbi, intelligence, russia, campaign, comey, document, committee, evidence, government, source, time, agency, hillary_clinton   \n",
       "...                                                                                                                                                                                                                   ...   \n",
       "26                                                        people, medium, thing, news, time, story, breitbart_news, twitter, show, question, interview, lot, fact, breitbart, fox_news, asked, talk, reporter, tweet, guy   \n",
       "5       trump, president, obama, white_house, donald_trump, administration, day, american, washington, meeting, office, policy, president_barack, national_security, presidency, friday, issue, election, told, statement   \n",
       "38                                                       people, black, america, american, white, history, church, world, christian, community, political, word, movement, nation, speech, country, life, group, gay, god   \n",
       "45                                                               american, job, people, worker, country, work, tax, year, policy, economy, economic, make, trade, america, good, plan, world, government, system, problem   \n",
       "28                                                                                                time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small   \n",
       "\n",
       "                                                                    Title  \\\n",
       "Papers                                                                      \n",
       "28                                       Could Cancer Drugs Treat Autism?   \n",
       "45                        Obamacare Repeal Moves a Step Closer to Reality   \n",
       "30      Resistance to the Antibiotic of Last Resort Is Silently Spreading   \n",
       "38                                         The Atlantic’s Week in Culture   \n",
       "5                                    Trump: ’I Think’ Hacking Was Russian   \n",
       "...                                                                   ...   \n",
       "26                                        Is Obamacare Just Bad Branding?   \n",
       "5                                    Trump: ’I Think’ Hacking Was Russian   \n",
       "38                                         The Atlantic’s Week in Culture   \n",
       "45                        Obamacare Repeal Moves a Step Closer to Reality   \n",
       "28                                       Could Cancer Drugs Treat Autism?   \n",
       "\n",
       "                                                                                                                                                                                                      Paper Desc  \n",
       "Papers                                                                                                                                                                                                            \n",
       "28      five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau  \n",
       "45      republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i  \n",
       "30      the alarm bells sounded on november 18, 2015. antibiotic resistance is usually a   crisis, one of the reasons its danger can be hard to convey. one by one, over the years, the drugs used to fight the   \n",
       "38      don’t miss, la la land’s   nostalgia —  david sims analyzes the nature of time in damien chazelle’s film, which pays homage to old hollywood while portraying the limits of worshipping the past. filmpa  \n",
       "5       updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus  \n",
       "...                                                                                                                                                                                                          ...  \n",
       "26      the senate began the process of dismantling obamacare early thursday morning, passing a budget blueprint that sets the stage for a formal repeal vote, potentially within a few weeks. the step is, for   \n",
       "5       updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus  \n",
       "38      don’t miss, la la land’s   nostalgia —  david sims analyzes the nature of time in damien chazelle’s film, which pays homage to old hollywood while portraying the limits of worshipping the past. filmpa  \n",
       "45      republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i  \n",
       "28      five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau  \n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df = pd.DataFrame()\n",
    "results_df['Papers'] = range(1, len(new_pre_papers)+1)\n",
    "results_df['Dominant Topics'] = [[topic_num+1 for topic_num, wt in item] for item in topic_preds]\n",
    "res = results_df.set_index(['Papers'])['Dominant Topics'].apply(pd.Series).stack().reset_index(level=1, drop=True)\n",
    "results_df = pd.DataFrame({'Dominant Topics': res.values}, index=res.index)\n",
    "results_df['Contribution %'] = [topic_wt for topic_list in \n",
    "                                        [[round(wt*100, 2) \n",
    "                                              for topic_num, wt in item] \n",
    "                                                 for item in topic_preds] \n",
    "                                    for topic_wt in topic_list]\n",
    "\n",
    "results_df['Topic Desc'] = [topics_df.iloc[t-1]['Terms per Topic'] for t in results_df['Dominant Topics'].values]\n",
    "results_df['Title'] = [new_pre_titles[i-1][:200] for i in results_df.index.values]\n",
    "results_df['Paper Desc'] = [new_pre_papers[i-1][:200] for i in results_df.index.values]\n",
    "\n",
    "\n",
    "pd.set_option('display.max_colwidth', 300)\n",
    "\n",
    "results_df.sort_values(by='Contribution %', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "formats": "py:light,ipynb",
   "notebook_metadata_filter": "-all"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}