0502_READING_NEW_PAPERS_MALLET.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = pd.read_csv('data/articles1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>publication</th>\n",
       "      <th>author</th>\n",
       "      <th>date</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>url</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>53293</td>\n",
       "      <td>73471</td>\n",
       "      <td>Patriots Day Is Best When It Digs Past the Her...</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>David Sims</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Patriots Day, Peter Berg’s new thriller that r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>53294</td>\n",
       "      <td>73472</td>\n",
       "      <td>A Break in the Search for the Origin of Comple...</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Ed Yong</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>In Norse mythology, humans and our world were ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>53295</td>\n",
       "      <td>73474</td>\n",
       "      <td>Obama’s Ingenious Mention of Atticus Finch</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Spencer Kornhaber</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>“If our democracy is to work in this increasin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53296</td>\n",
       "      <td>73475</td>\n",
       "      <td>Donald Trump Meets, and Assails, the Press</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>David A. Graham</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Updated on January 11 at 5:05 p. m. In his fir...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>53297</td>\n",
       "      <td>73476</td>\n",
       "      <td>Trump: ’I Think’ Hacking Was Russian</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Kaveh Waddell</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Updated at 12:25 p. m. After months of equivoc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0     id                                              title  \\\n",
       "0       53293  73471  Patriots Day Is Best When It Digs Past the Her...   \n",
       "1       53294  73472  A Break in the Search for the Origin of Comple...   \n",
       "2       53295  73474         Obama’s Ingenious Mention of Atticus Finch   \n",
       "3       53296  73475         Donald Trump Meets, and Assails, the Press   \n",
       "4       53297  73476               Trump: ’I Think’ Hacking Was Russian   \n",
       "\n",
       "  publication             author        date    year  month  url  \\\n",
       "0    Atlantic         David Sims  2017-01-11  2017.0    1.0  NaN   \n",
       "1    Atlantic            Ed Yong  2017-01-11  2017.0    1.0  NaN   \n",
       "2    Atlantic  Spencer Kornhaber  2017-01-11  2017.0    1.0  NaN   \n",
       "3    Atlantic    David A. Graham  2017-01-11  2017.0    1.0  NaN   \n",
       "4    Atlantic      Kaveh Waddell  2017-01-11  2017.0    1.0  NaN   \n",
       "\n",
       "                                             content  \n",
       "0  Patriots Day, Peter Berg’s new thriller that r...  \n",
       "1  In Norse mythology, humans and our world were ...  \n",
       "2  “If our democracy is to work in this increasin...  \n",
       "3  Updated on January 11 at 5:05 p. m. In his fir...  \n",
       "4  Updated at 12:25 p. m. After months of equivoc...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2 = pd.read_csv('data/articles2.csv')\n",
    "df_2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### let's select the first 50 new papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_titles = df_2['title'][:50].array\n",
    "new_papers = df_2['content'][:50].array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Trump Thanks L.L. Bean, the Latest Retailer Caught Between Him and His Critics'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_titles[34]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import nltk\n",
    "import gensim\n",
    "\n",
    "dictionary = gensim.corpora.Dictionary.load('models/dictionary.gensim')\n",
    "\n",
    "with open(\"lists/bow_corpus.txt\", \"rb\") as fp:   # Unpickling\n",
    "    bow_corpus = pickle.load(fp)\n",
    "\n",
    "with open(\"lists/norm_corpus_bigrams.txt\", \"rb\") as fp:\n",
    "    norm_corpus_bigrams = pickle.load(fp)\n",
    "\n",
    "with open(\"lists/norm_papers.txt\", \"rb\") as fp:\n",
    "    norm_papers = pickle.load(fp)\n",
    "\n",
    "with open(\"lists/pre_papers.txt\", \"rb\") as fp:\n",
    "    pre_papers = pickle.load(fp)\n",
    "\n",
    "with open(\"lists/pre_titles.txt\", \"rb\") as fp:\n",
    "    pre_titles = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PREPROCESS NEW PAPERS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "first preprcoess these new papers and extract features using the same sequence of steps we followed when building the topic models.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.98 ms, sys: 2.28 ms, total: 4.26 ms\n",
      "Wall time: 4.78 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import nltk\n",
    "\n",
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "wtk = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "wnl = nltk.stem.wordnet.WordNetLemmatizer()\n",
    "\n",
    "def normalise_corpus(papers, titles):\n",
    "    norm_papers = []\n",
    "    pre_papers = []\n",
    "    pre_titles = []\n",
    "    for i in range(len(papers)):\n",
    "        paper = papers[i]\n",
    "        title = titles[i]\n",
    "\n",
    "        paper = paper.lower()\n",
    "        paper_tokens = [token.strip() for token in wtk.tokenize(paper)]\n",
    "        paper_tokens = [wnl.lemmatize(token) for token in paper_tokens if not token.isnumeric()]\n",
    "        paper_tokens = [token for token in paper_tokens if len(token) > 1]\n",
    "        paper_tokens = [token for token in paper_tokens if token not in stop_words]\n",
    "        paper_tokens = list(filter(None, paper_tokens))\n",
    "\n",
    "        if paper_tokens:\n",
    "            norm_papers.append(paper_tokens)\n",
    "            pre_papers.append(paper)\n",
    "            pre_titles.append(title)\n",
    "\n",
    "    return norm_papers, pre_papers, pre_titles\n",
    "\n",
    "# we have pre_papers and pre_titles because the normalizing function removes empty papers and titles\n",
    "# so for consistency the papers and titles that we perform LDA on will be kept"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "let's create a text wrangling and feature engineering pipeline, which should match the same steps we followed when training our topic model.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_model = gensim.models.phrases.Phraser.load('models/bigram_model.gensim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_preprocessing_pipeline(documents, normaliser_fn, bigram_model, titles):\n",
    "    norm_docs, pre_papers, pre_titles = normaliser_fn(documents, titles)\n",
    "    norm_docs_bigrams = bigram_model[norm_docs]\n",
    "    return norm_docs_bigrams, pre_papers, pre_titles\n",
    "\n",
    "def bow_features_pipeline(tokenized_docs, dictionary):\n",
    "    paper_bow_features = [dictionary.doc2bow(text)\n",
    "                              for text in tokenized_docs]\n",
    "    return paper_bow_features\n",
    "\n",
    "norm_new_papers, new_pre_papers, new_pre_titles = text_preprocessing_pipeline(documents=new_papers,\n",
    "                                                                    normaliser_fn=normalise_corpus,\n",
    "                                                                    bigram_model=bigram_model,\n",
    "                                                                    titles=new_titles)\n",
    "\n",
    "norm_bow_features = bow_features_pipeline(tokenized_docs=norm_new_papers,\n",
    "                                         dictionary=dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['patriot', 'day', 'peter', 'berg', 'new', 'thriller', 'recreates', 'boston_marathon', 'bombing', 'ensuing', 'manhunt', 'followed', 'surprisingly', 'oblique', 'morally', 'ambiguous', 'movie', 'typically', 'straightforward', 'filmmaker', 'patriot', 'day', 'take', 'unexpectedly', 'cynical', 'view', 'chaos', 'rash', 'bureaucratic', 'infighting']\n"
     ]
    }
   ],
   "source": [
    "print(norm_new_papers[0][:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(22, 1), (26, 1), (31, 3), (36, 2), (39, 2), (42, 1), (57, 1), (69, 1), (85, 2), (96, 1), (97, 1), (108, 1), (140, 1), (144, 1), (149, 1), (157, 1), (167, 1), (168, 2), (174, 2), (177, 2), (183, 1), (230, 1), (236, 1), (237, 1), (240, 1), (253, 1), (261, 2), (266, 2), (280, 1), (287, 1)]\n"
     ]
    }
   ],
   "source": [
    "print(norm_bow_features[0][:30])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "### LOAD MALLET MODEL AND MAKE PREDICTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOPICS = 25\n",
    "\n",
    "load_lda_model = gensim.models.wrappers.LdaMallet.load('models/mallet/model_'+str(TOPICS)+'.gensim')\n",
    "\n",
    "# convert the ldaMallet to LdaModel. It was the only way to get some result with loading mallet model.\n",
    "load_lda_model = gensim.models.wrappers.ldamallet.malletmodel2ldamodel(load_lda_model)\n",
    "\n",
    "topics = [[(term, round(wt, 3))\n",
    "               for term, wt in load_lda_model.show_topic(n, topn=20)]\n",
    "                   for n in range(0, load_lda_model.num_topics)]\n",
    "\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "\n",
    "topics_df = pd.DataFrame([', '.join([term for term, wt in topic])\n",
    "                              for topic in topics],\n",
    "                         columns = ['Terms per Topic'],\n",
    "                         index=['Topic'+str(t) for t in range(1, load_lda_model.num_topics+1)]\n",
    "                         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Terms per Topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Topic1</th>\n",
       "      <td>game, team, player, sport, time, year, play, win, season, final, back, world, point, fan, athlete, match, played, league, run, day</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic2</th>\n",
       "      <td>official, email, investigation, report, clinton, information, russian, fbi, intelligence, russia, campaign, comey, document, committee, evidence, government, source, time, agency, hillary_clinton</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic3</th>\n",
       "      <td>military, isi, attack, syria, force, war, group, government, official, killed, islamic_state, american, syrian, iraq, people, soldier, city, civilian, fighting, army</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic4</th>\n",
       "      <td>law, state, court, decision, rule, federal, case, order, supreme_court, legal, public, government, policy, issue, judge, abortion, ban, justice, statement, action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic5</th>\n",
       "      <td>water, area, people, fire, flight, plane, cnn, home, official, land, airport, state, day, passenger, storm, air, island, road, part, ship</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic6</th>\n",
       "      <td>country, china, united_state, government, russia, iran, leader, north_korea, official, international, deal, russian, israel, world, putin, power, president, military, nation, japan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic7</th>\n",
       "      <td>trump, president, obama, white_house, donald_trump, administration, day, american, washington, meeting, office, policy, president_barack, national_security, presidency, friday, issue, election, told, statement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic8</th>\n",
       "      <td>muslim, attack, people, country, europe, britain, migrant, france, british, french, european, germany, london, german, islam, paris, report, uk, eu, european_union</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic9</th>\n",
       "      <td>company, facebook, car, technology, user, apple, online, google, product, internet, service, employee, twitter, customer, store, site, model, brand, time, tesla</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic10</th>\n",
       "      <td>million, company, money, year, business, billion, market, chinese, price, bank, financial, pay, fund, deal, percent, cost, investor, firm, paid, sale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic11</th>\n",
       "      <td>mr, time, interview, political, new_york, campaign, year, public, week, long, role, called, made, wrote, recent, month, official, work, including, question</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic12</th>\n",
       "      <td>republican, bill, democrat, house, senate, congress, ryan, senator, vote, plan, session, conservative, member, committee, people, health_care, american, obamacare, support, president</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic13</th>\n",
       "      <td>study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic14</th>\n",
       "      <td>people, medium, thing, news, time, story, breitbart_news, twitter, show, question, interview, lot, fact, breitbart, fox_news, asked, talk, reporter, tweet, guy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic15</th>\n",
       "      <td>time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic16</th>\n",
       "      <td>show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic17</th>\n",
       "      <td>woman, family, child, school, student, life, university, father, mother, parent, friend, told, home, son, college, year, young, girl, time, daughter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic18</th>\n",
       "      <td>case, charge, prison, court, told, victim, prosecutor, time, year, crime, woman, lawyer, trial, accused, judge, charged, attorney, statement, office, state</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic19</th>\n",
       "      <td>american, job, people, worker, country, work, tax, year, policy, economy, economic, make, trade, america, good, plan, world, government, system, problem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic20</th>\n",
       "      <td>party, election, percent, state, vote, voter, poll, support, political, democrat, result, point, candidate, win, republican, lead, race, campaign, national, number</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic21</th>\n",
       "      <td>city, people, new_york, event, day, protest, street, march, group, protester, building, time, local, park, california, home, mayor, crowd, center, place</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic22</th>\n",
       "      <td>trump, clinton, campaign, donald_trump, hillary_clinton, cruz, candidate, republican, sander, rubio, debate, supporter, hillary, speech, convention, rally, primary, support, delegate, win</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic23</th>\n",
       "      <td>immigration, country, united_state, mexico, refugee, immigrant, border, texas, reported, official, state, agent, security, breitbart_texas, number, report, mexican, program, national, american</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic24</th>\n",
       "      <td>people, black, america, american, white, history, church, world, christian, community, political, word, movement, nation, speech, country, life, group, gay, god</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Topic25</th>\n",
       "      <td>police, officer, man, gun, shooting, video, shot, police_officer, incident, suspect, killed, victim, death, law_enforcement, car, reported, authority, chief, report, attack</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                           Terms per Topic\n",
       "Topic1   game, team, player, sport, time, year, play, win, season, final, back, world, point, fan, athlete, match, played, league, run, day                                                                               \n",
       "Topic2   official, email, investigation, report, clinton, information, russian, fbi, intelligence, russia, campaign, comey, document, committee, evidence, government, source, time, agency, hillary_clinton              \n",
       "Topic3   military, isi, attack, syria, force, war, group, government, official, killed, islamic_state, american, syrian, iraq, people, soldier, city, civilian, fighting, army                                            \n",
       "Topic4   law, state, court, decision, rule, federal, case, order, supreme_court, legal, public, government, policy, issue, judge, abortion, ban, justice, statement, action                                               \n",
       "Topic5   water, area, people, fire, flight, plane, cnn, home, official, land, airport, state, day, passenger, storm, air, island, road, part, ship                                                                        \n",
       "Topic6   country, china, united_state, government, russia, iran, leader, north_korea, official, international, deal, russian, israel, world, putin, power, president, military, nation, japan                             \n",
       "Topic7   trump, president, obama, white_house, donald_trump, administration, day, american, washington, meeting, office, policy, president_barack, national_security, presidency, friday, issue, election, told, statement\n",
       "Topic8   muslim, attack, people, country, europe, britain, migrant, france, british, french, european, germany, london, german, islam, paris, report, uk, eu, european_union                                              \n",
       "Topic9   company, facebook, car, technology, user, apple, online, google, product, internet, service, employee, twitter, customer, store, site, model, brand, time, tesla                                                 \n",
       "Topic10  million, company, money, year, business, billion, market, chinese, price, bank, financial, pay, fund, deal, percent, cost, investor, firm, paid, sale                                                            \n",
       "Topic11  mr, time, interview, political, new_york, campaign, year, public, week, long, role, called, made, wrote, recent, month, official, work, including, question                                                      \n",
       "Topic12  republican, bill, democrat, house, senate, congress, ryan, senator, vote, plan, session, conservative, member, committee, people, health_care, american, obamacare, support, president                           \n",
       "Topic13  study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result                                                       \n",
       "Topic14  people, medium, thing, news, time, story, breitbart_news, twitter, show, question, interview, lot, fact, breitbart, fox_news, asked, talk, reporter, tweet, guy                                                  \n",
       "Topic15  time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small                                                                                          \n",
       "Topic16  show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story                                                                \n",
       "Topic17  woman, family, child, school, student, life, university, father, mother, parent, friend, told, home, son, college, year, young, girl, time, daughter                                                             \n",
       "Topic18  case, charge, prison, court, told, victim, prosecutor, time, year, crime, woman, lawyer, trial, accused, judge, charged, attorney, statement, office, state                                                      \n",
       "Topic19  american, job, people, worker, country, work, tax, year, policy, economy, economic, make, trade, america, good, plan, world, government, system, problem                                                         \n",
       "Topic20  party, election, percent, state, vote, voter, poll, support, political, democrat, result, point, candidate, win, republican, lead, race, campaign, national, number                                              \n",
       "Topic21  city, people, new_york, event, day, protest, street, march, group, protester, building, time, local, park, california, home, mayor, crowd, center, place                                                         \n",
       "Topic22  trump, clinton, campaign, donald_trump, hillary_clinton, cruz, candidate, republican, sander, rubio, debate, supporter, hillary, speech, convention, rally, primary, support, delegate, win                      \n",
       "Topic23  immigration, country, united_state, mexico, refugee, immigrant, border, texas, reported, official, state, agent, security, breitbart_texas, number, report, mexican, program, national, american                 \n",
       "Topic24  people, black, america, american, white, history, church, world, christian, community, political, word, movement, nation, speech, country, life, group, gay, god                                                 \n",
       "Topic25  police, officer, man, gun, shooting, video, shot, police_officer, incident, suspect, killed, victim, death, law_enforcement, car, reported, authority, chief, report, attack                                     "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_topic_predictions(topic_model, corpus, topn=3):\n",
    "    topic_predictions = topic_model[corpus]\n",
    "    best_topics = [[(topic, round(wt, 3))\n",
    "                        for topic, wt in sorted(topic_predictions[i],\n",
    "                                                key=lambda row: -row[1])[:topn]]\n",
    "                            for i in range(len(topic_predictions))]\n",
    "    return best_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_preds = get_topic_predictions(topic_model=load_lda_model,\n",
    "                                    corpus=norm_bow_features, topn=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.DataFrame()\n",
    "results_df['Papers'] = range(1, len(new_pre_papers)+1)\n",
    "results_df['Dominant Topics'] = [[topic_num+1 for topic_num, wt in item] for item in topic_preds]\n",
    "res = results_df.set_index(['Papers'])['Dominant Topics'].apply(pd.Series).stack().reset_index(level=1, drop=True)\n",
    "results_df = pd.DataFrame({'Dominant Topics': res.values}, index=res.index)\n",
    "results_df['Contribution %'] = [topic_wt for topic_list in \n",
    "                                        [[round(wt*100, 2) \n",
    "                                              for topic_num, wt in item] \n",
    "                                                 for item in topic_preds] \n",
    "                                    for topic_wt in topic_list]\n",
    "\n",
    "results_df['Topic Desc'] = [topics_df.iloc[t-1]['Terms per Topic'] for t in results_df['Dominant Topics'].values]\n",
    "results_df['Title'] = [new_pre_titles[i-1][:200] for i in results_df.index.values]\n",
    "results_df['Paper Desc'] = [new_pre_papers[i-1][:200] for i in results_df.index.values]\n",
    "\n",
    "\n",
    "pd.set_option('display.max_colwidth', 300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dominant Topics</th>\n",
       "      <th>Contribution %</th>\n",
       "      <th>Topic Desc</th>\n",
       "      <th>Title</th>\n",
       "      <th>Paper Desc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Papers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16</td>\n",
       "      <td>17.9</td>\n",
       "      <td>show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story</td>\n",
       "      <td>Patriots Day Is Best When It Digs Past the Heroism</td>\n",
       "      <td>patriots day, peter berg’s new thriller that recreates the 2013 boston marathon bombing and the ensuing manhunt that followed it, is a surprisingly oblique, morally ambiguous movie from a typically st</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15</td>\n",
       "      <td>15.7</td>\n",
       "      <td>time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small</td>\n",
       "      <td>Patriots Day Is Best When It Digs Past the Heroism</td>\n",
       "      <td>patriots day, peter berg’s new thriller that recreates the 2013 boston marathon bombing and the ensuing manhunt that followed it, is a surprisingly oblique, morally ambiguous movie from a typically st</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13</td>\n",
       "      <td>24.6</td>\n",
       "      <td>study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result</td>\n",
       "      <td>A Break in the Search for the Origin of Complex Life</td>\n",
       "      <td>in norse mythology, humans and our world were created by a pantheon of gods who lived in the realm of asgard. as it turns out, these stories have a grain of truth to them. thanks to a team of scientis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>21.3</td>\n",
       "      <td>time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small</td>\n",
       "      <td>A Break in the Search for the Origin of Complex Life</td>\n",
       "      <td>in norse mythology, humans and our world were created by a pantheon of gods who lived in the realm of asgard. as it turns out, these stories have a grain of truth to them. thanks to a team of scientis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>34.2</td>\n",
       "      <td>people, black, america, american, white, history, church, world, christian, community, political, word, movement, nation, speech, country, life, group, gay, god</td>\n",
       "      <td>Obama’s Ingenious Mention of Atticus Finch</td>\n",
       "      <td>“if our democracy is to work in this increasingly diverse nation,” barack obama said in his farewell address last night, “each one of us must try to heed the advice of one of the great characters in a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>12</td>\n",
       "      <td>11.8</td>\n",
       "      <td>republican, bill, democrat, house, senate, congress, ryan, senator, vote, plan, session, conservative, member, committee, people, health_care, american, obamacare, support, president</td>\n",
       "      <td>Sessions Exaggerated His Record on Gun Prosecutions</td>\n",
       "      <td>in senator jeff sessions of alabama, opponents of new firearms restrictions have an attorney general nominee ready to implement their frequent call to “enforce the laws on the books. ” during his conf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>2</td>\n",
       "      <td>25.0</td>\n",
       "      <td>official, email, investigation, report, clinton, information, russian, fbi, intelligence, russia, campaign, comey, document, committee, evidence, government, source, time, agency, hillary_clinton</td>\n",
       "      <td>Why Elections Are Now Classified as ’Critical Infrastructure’</td>\n",
       "      <td>last friday, the same day three of the top spy agencies in the u. s. released a summary of an investigation into russia’s role in cyberattacks before the election, the department of homeland security</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>4</td>\n",
       "      <td>10.2</td>\n",
       "      <td>law, state, court, decision, rule, federal, case, order, supreme_court, legal, public, government, policy, issue, judge, abortion, ban, justice, statement, action</td>\n",
       "      <td>Why Elections Are Now Classified as ’Critical Infrastructure’</td>\n",
       "      <td>last friday, the same day three of the top spy agencies in the u. s. released a summary of an investigation into russia’s role in cyberattacks before the election, the department of homeland security</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>15</td>\n",
       "      <td>31.1</td>\n",
       "      <td>time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small</td>\n",
       "      <td>20th Century Women Is an Ode to Female Resilience</td>\n",
       "      <td>in a scene early on in 20th century women, dorothea (annette bening) helps her son construct a birthday cake in the kitchen, plugging candles into a mess of whipped cream and strawberries. “wait a few</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>16</td>\n",
       "      <td>19.1</td>\n",
       "      <td>show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story</td>\n",
       "      <td>20th Century Women Is an Ode to Female Resilience</td>\n",
       "      <td>in a scene early on in 20th century women, dorothea (annette bening) helps her son construct a birthday cake in the kitchen, plugging candles into a mess of whipped cream and strawberries. “wait a few</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Dominant Topics  Contribution %  \\\n",
       "Papers                                    \n",
       "1                    16            17.9   \n",
       "1                    15            15.7   \n",
       "2                    13            24.6   \n",
       "2                    15            21.3   \n",
       "3                    24            34.2   \n",
       "...                 ...             ...   \n",
       "48                   12            11.8   \n",
       "49                    2            25.0   \n",
       "49                    4            10.2   \n",
       "50                   15            31.1   \n",
       "50                   16            19.1   \n",
       "\n",
       "                                                                                                                                                                                                 Topic Desc  \\\n",
       "Papers                                                                                                                                                                                                        \n",
       "1                                                         show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story   \n",
       "1                                                                                   time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small   \n",
       "2                                                study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result   \n",
       "2                                                                                   time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small   \n",
       "3                                          people, black, america, american, white, history, church, world, christian, community, political, word, movement, nation, speech, country, life, group, gay, god   \n",
       "...                                                                                                                                                                                                     ...   \n",
       "48                   republican, bill, democrat, house, senate, congress, ryan, senator, vote, plan, session, conservative, member, committee, people, health_care, american, obamacare, support, president   \n",
       "49      official, email, investigation, report, clinton, information, russian, fbi, intelligence, russia, campaign, comey, document, committee, evidence, government, source, time, agency, hillary_clinton   \n",
       "49                                       law, state, court, decision, rule, federal, case, order, supreme_court, legal, public, government, policy, issue, judge, abortion, ban, justice, statement, action   \n",
       "50                                                                                  time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small   \n",
       "50                                                        show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story   \n",
       "\n",
       "                                                                Title  \\\n",
       "Papers                                                                  \n",
       "1                  Patriots Day Is Best When It Digs Past the Heroism   \n",
       "1                  Patriots Day Is Best When It Digs Past the Heroism   \n",
       "2                A Break in the Search for the Origin of Complex Life   \n",
       "2                A Break in the Search for the Origin of Complex Life   \n",
       "3                          Obama’s Ingenious Mention of Atticus Finch   \n",
       "...                                                               ...   \n",
       "48                Sessions Exaggerated His Record on Gun Prosecutions   \n",
       "49      Why Elections Are Now Classified as ’Critical Infrastructure’   \n",
       "49      Why Elections Are Now Classified as ’Critical Infrastructure’   \n",
       "50                  20th Century Women Is an Ode to Female Resilience   \n",
       "50                  20th Century Women Is an Ode to Female Resilience   \n",
       "\n",
       "                                                                                                                                                                                                      Paper Desc  \n",
       "Papers                                                                                                                                                                                                            \n",
       "1       patriots day, peter berg’s new thriller that recreates the 2013 boston marathon bombing and the ensuing manhunt that followed it, is a surprisingly oblique, morally ambiguous movie from a typically st  \n",
       "1       patriots day, peter berg’s new thriller that recreates the 2013 boston marathon bombing and the ensuing manhunt that followed it, is a surprisingly oblique, morally ambiguous movie from a typically st  \n",
       "2       in norse mythology, humans and our world were created by a pantheon of gods who lived in the realm of asgard. as it turns out, these stories have a grain of truth to them. thanks to a team of scientis  \n",
       "2       in norse mythology, humans and our world were created by a pantheon of gods who lived in the realm of asgard. as it turns out, these stories have a grain of truth to them. thanks to a team of scientis  \n",
       "3       “if our democracy is to work in this increasingly diverse nation,” barack obama said in his farewell address last night, “each one of us must try to heed the advice of one of the great characters in a  \n",
       "...                                                                                                                                                                                                          ...  \n",
       "48      in senator jeff sessions of alabama, opponents of new firearms restrictions have an attorney general nominee ready to implement their frequent call to “enforce the laws on the books. ” during his conf  \n",
       "49      last friday, the same day three of the top spy agencies in the u. s. released a summary of an investigation into russia’s role in cyberattacks before the election, the department of homeland security   \n",
       "49      last friday, the same day three of the top spy agencies in the u. s. released a summary of an investigation into russia’s role in cyberattacks before the election, the department of homeland security   \n",
       "50      in a scene early on in 20th century women, dorothea (annette bening) helps her son construct a birthday cake in the kitchen, plugging candles into a mess of whipped cream and strawberries. “wait a few  \n",
       "50      in a scene early on in 20th century women, dorothea (annette bening) helps her son construct a birthday cake in the kitchen, plugging candles into a mess of whipped cream and strawberries. “wait a few  \n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dominant Topics</th>\n",
       "      <th>Contribution %</th>\n",
       "      <th>Topic Desc</th>\n",
       "      <th>Title</th>\n",
       "      <th>Paper Desc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Papers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>13</td>\n",
       "      <td>67.4</td>\n",
       "      <td>study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result</td>\n",
       "      <td>Could Cancer Drugs Treat Autism?</td>\n",
       "      <td>five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>12</td>\n",
       "      <td>58.8</td>\n",
       "      <td>republican, bill, democrat, house, senate, congress, ryan, senator, vote, plan, session, conservative, member, committee, people, health_care, american, obamacare, support, president</td>\n",
       "      <td>Obamacare Repeal Moves a Step Closer to Reality</td>\n",
       "      <td>republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>13</td>\n",
       "      <td>52.7</td>\n",
       "      <td>study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result</td>\n",
       "      <td>Resistance to the Antibiotic of Last Resort Is Silently Spreading</td>\n",
       "      <td>the alarm bells sounded on november 18, 2015. antibiotic resistance is usually a   crisis, one of the reasons its danger can be hard to convey. one by one, over the years, the drugs used to fight the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>16</td>\n",
       "      <td>51.1</td>\n",
       "      <td>show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story</td>\n",
       "      <td>The Atlantic’s Week in Culture</td>\n",
       "      <td>don’t miss, la la land’s   nostalgia —  david sims analyzes the nature of time in damien chazelle’s film, which pays homage to old hollywood while portraying the limits of worshipping the past. filmpa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>47.2</td>\n",
       "      <td>official, email, investigation, report, clinton, information, russian, fbi, intelligence, russia, campaign, comey, document, committee, evidence, government, source, time, agency, hillary_clinton</td>\n",
       "      <td>Trump: ’I Think’ Hacking Was Russian</td>\n",
       "      <td>updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>14</td>\n",
       "      <td>7.3</td>\n",
       "      <td>people, medium, thing, news, time, story, breitbart_news, twitter, show, question, interview, lot, fact, breitbart, fox_news, asked, talk, reporter, tweet, guy</td>\n",
       "      <td>Is Obamacare Just Bad Branding?</td>\n",
       "      <td>the senate began the process of dismantling obamacare early thursday morning, passing a budget blueprint that sets the stage for a formal repeal vote, potentially within a few weeks. the step is, for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7</td>\n",
       "      <td>6.8</td>\n",
       "      <td>trump, president, obama, white_house, donald_trump, administration, day, american, washington, meeting, office, policy, president_barack, national_security, presidency, friday, issue, election, told, statement</td>\n",
       "      <td>Trump: ’I Think’ Hacking Was Russian</td>\n",
       "      <td>updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>24</td>\n",
       "      <td>6.8</td>\n",
       "      <td>people, black, america, american, white, history, church, world, christian, community, political, word, movement, nation, speech, country, life, group, gay, god</td>\n",
       "      <td>The Atlantic’s Week in Culture</td>\n",
       "      <td>don’t miss, la la land’s   nostalgia —  david sims analyzes the nature of time in damien chazelle’s film, which pays homage to old hollywood while portraying the limits of worshipping the past. filmpa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>19</td>\n",
       "      <td>5.7</td>\n",
       "      <td>american, job, people, worker, country, work, tax, year, policy, economy, economic, make, trade, america, good, plan, world, government, system, problem</td>\n",
       "      <td>Obamacare Repeal Moves a Step Closer to Reality</td>\n",
       "      <td>republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>15</td>\n",
       "      <td>5.4</td>\n",
       "      <td>time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small</td>\n",
       "      <td>Could Cancer Drugs Treat Autism?</td>\n",
       "      <td>five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Dominant Topics  Contribution %  \\\n",
       "Papers                                    \n",
       "28                   13            67.4   \n",
       "45                   12            58.8   \n",
       "30                   13            52.7   \n",
       "38                   16            51.1   \n",
       "5                     2            47.2   \n",
       "...                 ...             ...   \n",
       "26                   14             7.3   \n",
       "5                     7             6.8   \n",
       "38                   24             6.8   \n",
       "45                   19             5.7   \n",
       "28                   15             5.4   \n",
       "\n",
       "                                                                                                                                                                                                               Topic Desc  \\\n",
       "Papers                                                                                                                                                                                                                      \n",
       "28                                                             study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result   \n",
       "45                                 republican, bill, democrat, house, senate, congress, ryan, senator, vote, plan, session, conservative, member, committee, people, health_care, american, obamacare, support, president   \n",
       "30                                                             study, health, people, dr, drug, case, doctor, patient, year, research, found, medical, risk, problem, hospital, treatment, researcher, test, time, result   \n",
       "38                                                                      show, film, star, movie, year, actor, character, song, music, series, love, play, hollywood, audience, book, time, fan, award, performance, story   \n",
       "5                     official, email, investigation, report, clinton, information, russian, fbi, intelligence, russia, campaign, comey, document, committee, evidence, government, source, time, agency, hillary_clinton   \n",
       "...                                                                                                                                                                                                                   ...   \n",
       "26                                                        people, medium, thing, news, time, story, breitbart_news, twitter, show, question, interview, lot, fact, breitbart, fox_news, asked, talk, reporter, tweet, guy   \n",
       "5       trump, president, obama, white_house, donald_trump, administration, day, american, washington, meeting, office, policy, president_barack, national_security, presidency, friday, issue, election, told, statement   \n",
       "38                                                       people, black, america, american, white, history, church, world, christian, community, political, word, movement, nation, speech, country, life, group, gay, god   \n",
       "45                                                               american, job, people, worker, country, work, tax, year, policy, economy, economic, make, trade, america, good, plan, world, government, system, problem   \n",
       "28                                                                                                time, work, thing, back, make, day, good, food, life, world, made, find, long, place, book, kind, art, hand, dog, small   \n",
       "\n",
       "                                                                    Title  \\\n",
       "Papers                                                                      \n",
       "28                                       Could Cancer Drugs Treat Autism?   \n",
       "45                        Obamacare Repeal Moves a Step Closer to Reality   \n",
       "30      Resistance to the Antibiotic of Last Resort Is Silently Spreading   \n",
       "38                                         The Atlantic’s Week in Culture   \n",
       "5                                    Trump: ’I Think’ Hacking Was Russian   \n",
       "...                                                                   ...   \n",
       "26                                        Is Obamacare Just Bad Branding?   \n",
       "5                                    Trump: ’I Think’ Hacking Was Russian   \n",
       "38                                         The Atlantic’s Week in Culture   \n",
       "45                        Obamacare Repeal Moves a Step Closer to Reality   \n",
       "28                                       Could Cancer Drugs Treat Autism?   \n",
       "\n",
       "                                                                                                                                                                                                      Paper Desc  \n",
       "Papers                                                                                                                                                                                                            \n",
       "28      five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau  \n",
       "45      republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i  \n",
       "30      the alarm bells sounded on november 18, 2015. antibiotic resistance is usually a   crisis, one of the reasons its danger can be hard to convey. one by one, over the years, the drugs used to fight the   \n",
       "38      don’t miss, la la land’s   nostalgia —  david sims analyzes the nature of time in damien chazelle’s film, which pays homage to old hollywood while portraying the limits of worshipping the past. filmpa  \n",
       "5       updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus  \n",
       "...                                                                                                                                                                                                          ...  \n",
       "26      the senate began the process of dismantling obamacare early thursday morning, passing a budget blueprint that sets the stage for a formal repeal vote, potentially within a few weeks. the step is, for   \n",
       "5       updated at 12:25 p. m. after months of equivocating on the origin of cyberattacks that targeted democrats before the election,   donald trump said wednesday that he thinks russia was behind the intrus  \n",
       "38      don’t miss, la la land’s   nostalgia —  david sims analyzes the nature of time in damien chazelle’s film, which pays homage to old hollywood while portraying the limits of worshipping the past. filmpa  \n",
       "45      republicans in congress can now repeal most of the affordable care act with a simple majority vote. the house on friday passed a budget resolution allowing congress to bypass a democratic filibuster i  \n",
       "28      five years ago, on charlie ryan’s second birthday, a big lump mysteriously formed on the side of his abdomen. at the emergency room his parents took him to, doctors suggested the lump was a hernia cau  \n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df.sort_values(by='Contribution %', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "formats": "py:light,ipynb",
   "notebook_metadata_filter": "-all"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}