Example_20 Newsgroups.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "970501a8",
   "metadata": {},
   "source": [
    "### Generation of 2D Scatterplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cad1303e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "\n",
    "import gensim\n",
    "import gensim.corpora as corpora\n",
    "from gensim.utils import simple_preprocess\n",
    "\n",
    "from scipy.spatial.distance import pdist\n",
    "\n",
    "import spacy\n",
    "from pprint import pprint\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# NLTK Stop words\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = stopwords.words('english')\n",
    "stop_words.extend(['from','subject','re','edu','use'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ea3acd69",
   "metadata": {},
   "outputs": [],
   "source": [
    "newsgroups_train = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5e24b9f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "I am sure some bashers of Pens fans are pretty confused about the lack\n",
      "of any kind of posts about the recent Pens massacre of the Devils. Actually,\n",
      "I am  bit puzzled too and a bit relieved. However, I am going to put an end\n",
      "to non-PIttsburghers' relief with a bit of praise for the Pens. Man, they\n",
      "are killing those Devils worse than I thought. Jagr just showed you why\n",
      "he is much better than his regular season stats. He is also a lot\n",
      "fo fun to watch in the playoffs. Bowman should let JAgr have a lot of\n",
      "fun in the next couple of games since the Pens are going to beat the pulp out of Jersey anyway. I was very disappointed not to see the Islanders lose the final\n",
      "regular season game.          PENS RULE!!!\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(newsgroups_train.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "576b276d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sent_to_words(sentences):\n",
    "    for sentence in sentences:\n",
    "        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ef0dd217",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
    "def remove_stopwords(texts):\n",
    "    return [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "63de78ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):\n",
    "    \"\"\"https://spacy.io/api/annotation\"\"\"\n",
    "    texts_out = []\n",
    "    for sent in texts:\n",
    "        doc = nlp(\" \".join(sent)) \n",
    "        texts_out.append([token.lemma_ for token in doc])\n",
    "    return texts_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "34bf7d11",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start removing stop words\n",
      "Installing spacy\n",
      "Start lemmatizing words\n"
     ]
    }
   ],
   "source": [
    "data = newsgroups_train.data\n",
    "data_words = list(sent_to_words(data))\n",
    "\n",
    "# Remove Stop Words\n",
    "print(\"Start removing stop words\")\n",
    "data_words_nostops = remove_stopwords(data_words)\n",
    "\n",
    "# Initialize spacy 'en' model, keeping only tagger component (for efficiency)\n",
    "# conda install -c conda-forge spacy-model-en_core_web_sm\n",
    "print(\"Installing spacy\")\n",
    "nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])\n",
    "\n",
    "# Do lemmatization keeping only noun, adj, vb, adv\n",
    "print(\"Start lemmatizing words\")\n",
    "#data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])\n",
    "data_lemmatized = lemmatization(data_words_nostops, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ed984679",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sure', 'basher', 'pen', 'fan', 'pretty', 'confused', 'lack', 'kind', 'post', 'recent', 'pen', 'massacre', 'devil', 'actually', 'bite', 'puzzled', 'bit', 'relieve', 'however', 'go', 'put', 'end', 'non', 'pittsburgher', 'relief', 'bit', 'praise', 'pen', 'man', 'kill', 'devil', 'bad', 'thought', 'jagr', 'show', 'much', 'well', 'regular', 'season', 'stat', 'also', 'lot', 'fo', 'fun', 'watch', 'playoff', 'bowman', 'let', 'jagr', 'lot', 'fun', 'next', 'couple', 'game', 'since', 'pen', 'go', 'beat', 'pulp', 'jersey', 'anyway', 'disappoint', 'see', 'islander', 'lose', 'final', 'regular', 'season', 'game', 'pen', 'rule']\n"
     ]
    }
   ],
   "source": [
    "print(data_lemmatized[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c94f0970",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lemmatized_min_length = []\n",
    "\n",
    "for sublist in data_lemmatized:\n",
    "    # Use a list comprehension to filter out strings with less than two characters\n",
    "    sublist = [word for word in sublist if len(word) > 3]\n",
    "    data_lemmatized_min_length.append(sublist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2e3753ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = newsgroups_train.target.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "631da1d4-725c-436a-9787-7523ab66c13d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alt.atheism',\n",
       " 'comp.graphics',\n",
       " 'comp.os.ms-windows.misc',\n",
       " 'comp.sys.ibm.pc.hardware',\n",
       " 'comp.sys.mac.hardware',\n",
       " 'comp.windows.x',\n",
       " 'misc.forsale',\n",
       " 'rec.autos',\n",
       " 'rec.motorcycles',\n",
       " 'rec.sport.baseball',\n",
       " 'rec.sport.hockey',\n",
       " 'sci.crypt',\n",
       " 'sci.electronics',\n",
       " 'sci.med',\n",
       " 'sci.space',\n",
       " 'soc.religion.christian',\n",
       " 'talk.politics.guns',\n",
       " 'talk.politics.mideast',\n",
       " 'talk.politics.misc',\n",
       " 'talk.religion.misc']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newsgroups_train.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "eb291370",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[(0, 1), (1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1), (8, 1), (9, 2), (10, 1), (11, 1), (12, 2), (13, 1), (14, 1), (15, 2), (16, 1), (17, 1), (18, 1), (19, 1), (20, 1), (21, 1), (22, 1), (23, 1), (24, 1), (25, 1), (26, 1), (27, 1), (28, 1), (29, 1), (30, 1), (31, 1), (32, 2), (33, 1), (34, 1), (35, 1), (36, 2), (37, 1), (38, 1), (39, 1), (40, 1), (41, 1), (42, 1), (43, 1)]]\n"
     ]
    }
   ],
   "source": [
    "# Create Dictionary\n",
    "id2word = corpora.Dictionary(data_lemmatized_min_length)\n",
    "\n",
    "# Create Corpus\n",
    "texts = data_lemmatized_min_length\n",
    "\n",
    "# Term Document Frequency\n",
    "corpus = [id2word.doc2bow(text) for text in texts]\n",
    "\n",
    "# View \n",
    "print(corpus[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bd905906",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import dok_matrix\n",
    "\n",
    "# Define function to convert Gensim corpus to a sparse pandas DataFrame\n",
    "def corpus_to_sparse_dataframe(corpus):\n",
    "    word_freq = dok_matrix((len(corpus), len(id2word)), dtype=int)\n",
    "\n",
    "    for i, doc in enumerate(corpus):\n",
    "        for word_id, freq in doc:\n",
    "            word_freq[i, word_id] = freq\n",
    "\n",
    "    dataframe = pd.DataFrame.sparse.from_spmatrix(word_freq)\n",
    "    dataframe.columns = [id2word[word_id] for word_id in range(len(id2word))]\n",
    "    return dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "78543c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "VSM = corpus_to_sparse_dataframe(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c1597ba3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.153*\"would\" + 0.128*\"know\" + 0.119*\"people\" + 0.117*\"like\" + 0.114*\"think\" + 0.106*\"make\" + 0.102*\"well\" + 0.101*\"good\" + 0.099*\"window\" + 0.098*\"drive\"'),\n",
       " (1,\n",
       "  '0.254*\"window\" + 0.230*\"drive\" + 0.218*\"card\" + 0.203*\"file\" + 0.165*\"thank\" + 0.141*\"driver\" + 0.139*\"disk\" + -0.135*\"people\" + 0.132*\"scsi\" + 0.109*\"program\"'),\n",
       " (2,\n",
       "  '0.505*\"drive\" + -0.292*\"window\" + -0.279*\"file\" + 0.271*\"scsi\" + 0.236*\"game\" + 0.140*\"controller\" + -0.131*\"program\" + 0.128*\"disk\" + 0.118*\"hard\" + 0.114*\"card\"'),\n",
       " (3,\n",
       "  '0.549*\"game\" + -0.271*\"drive\" + 0.241*\"team\" + 0.167*\"play\" + 0.159*\"player\" + -0.158*\"scsi\" + 0.126*\"hockey\" + 0.116*\"window\" + 0.115*\"baseball\" + 0.109*\"season\"'),\n",
       " (4,\n",
       "  '0.346*\"file\" + 0.325*\"window\" + 0.276*\"drive\" + -0.231*\"thank\" + -0.221*\"please\" + -0.193*\"mail\" + -0.176*\"chip\" + -0.176*\"card\" + 0.143*\"disk\" + 0.140*\"scsi\"'),\n",
       " (5,\n",
       "  '0.457*\"card\" + -0.268*\"drive\" + -0.260*\"thank\" + -0.236*\"please\" + 0.216*\"driver\" + 0.179*\"window\" + 0.178*\"video\" + -0.171*\"mail\" + 0.167*\"monitor\" + -0.161*\"file\"'),\n",
       " (6,\n",
       "  '-0.293*\"chip\" + 0.287*\"card\" + 0.186*\"jesus\" + -0.181*\"encryption\" + 0.178*\"thank\" + -0.175*\"clipper\" + 0.157*\"driver\" + -0.148*\"government\" + -0.135*\"phone\" + -0.128*\"file\"'),\n",
       " (7,\n",
       "  '-0.386*\"bike\" + 0.234*\"game\" + 0.202*\"card\" + 0.186*\"chip\" + -0.175*\"ride\" + 0.169*\"scsi\" + 0.156*\"file\" + -0.150*\"window\" + 0.116*\"encryption\" + -0.115*\"good\"'),\n",
       " (8,\n",
       "  '0.459*\"armenian\" + -0.184*\"chip\" + 0.168*\"israel\" + -0.161*\"jesus\" + 0.160*\"turkish\" + 0.114*\"muslim\" + 0.110*\"kill\" + -0.110*\"believe\" + 0.107*\"genocide\" + 0.105*\"arab\"'),\n",
       " (9,\n",
       "  '0.439*\"window\" + -0.283*\"file\" + -0.236*\"image\" + -0.195*\"monitor\" + 0.195*\"thank\" + 0.172*\"chip\" + 0.152*\"scsi\" + 0.146*\"driver\" + -0.143*\"color\" + -0.139*\"format\"'),\n",
       " (10,\n",
       "  '-0.407*\"file\" + 0.290*\"window\" + -0.215*\"card\" + -0.200*\"bike\" + -0.141*\"driver\" + -0.137*\"thank\" + -0.137*\"format\" + 0.132*\"sale\" + 0.127*\"offer\" + 0.125*\"price\"'),\n",
       " (11,\n",
       "  '0.321*\"file\" + 0.282*\"bike\" + -0.222*\"scsi\" + -0.172*\"image\" + -0.157*\"space\" + 0.143*\"chip\" + 0.133*\"sale\" + 0.130*\"jesus\" + 0.120*\"sell\" + 0.114*\"phone\"'),\n",
       " (12,\n",
       "  '0.372*\"monitor\" + -0.354*\"scsi\" + -0.311*\"driver\" + 0.185*\"thank\" + -0.178*\"card\" + 0.139*\"problem\" + -0.137*\"list\" + 0.132*\"drive\" + 0.129*\"color\" + -0.122*\"bike\"'),\n",
       " (13,\n",
       "  '-0.273*\"pitt\" + -0.268*\"bank\" + -0.265*\"cadre\" + -0.264*\"chastity\" + -0.261*\"skepticism\" + -0.261*\"intellect\" + -0.260*\"shameful\" + -0.257*\"surrender\" + -0.255*\"gordon\" + -0.202*\"soon\"'),\n",
       " (14,\n",
       "  '0.360*\"armenian\" + 0.241*\"chip\" + -0.183*\"israel\" + 0.172*\"jesus\" + 0.165*\"bike\" + -0.151*\"right\" + 0.139*\"scsi\" + -0.116*\"monitor\" + 0.113*\"turkish\" + 0.112*\"pitt\"'),\n",
       " (15,\n",
       "  '-0.315*\"port\" + -0.282*\"modem\" + 0.253*\"window\" + -0.230*\"problem\" + 0.208*\"monitor\" + 0.203*\"drive\" + -0.195*\"printer\" + 0.195*\"color\" + -0.158*\"serial\" + 0.150*\"image\"'),\n",
       " (16,\n",
       "  '0.387*\"israel\" + 0.354*\"scsi\" + 0.217*\"bike\" + 0.198*\"arab\" + -0.165*\"space\" + 0.161*\"israeli\" + 0.160*\"image\" + -0.158*\"armenian\" + -0.157*\"card\" + 0.157*\"jews\"'),\n",
       " (17,\n",
       "  '-0.360*\"printer\" + -0.326*\"driver\" + -0.247*\"font\" + 0.241*\"bike\" + 0.220*\"game\" + 0.198*\"card\" + -0.168*\"print\" + -0.160*\"team\" + 0.149*\"israel\" + 0.137*\"system\"'),\n",
       " (18,\n",
       "  '0.320*\"driver\" + 0.290*\"game\" + -0.281*\"scsi\" + -0.212*\"file\" + -0.205*\"team\" + -0.182*\"monitor\" + 0.178*\"printer\" + -0.170*\"player\" + 0.162*\"disk\" + -0.158*\"window\"'),\n",
       " (19,\n",
       "  '-0.370*\"scsi\" + -0.288*\"game\" + 0.267*\"drive\" + 0.230*\"team\" + 0.180*\"israel\" + 0.179*\"player\" + -0.128*\"fire\" + -0.125*\"batf\" + 0.115*\"card\" + -0.112*\"koresh\"')]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from gensim.models import TfidfModel\n",
    "from gensim.models import LsiModel\n",
    "\n",
    "model = TfidfModel(corpus)  # fit model\n",
    "tfidf_corpus = model[corpus]\n",
    "\n",
    "K = 20\n",
    "tfidf_lsi_model = LsiModel(tfidf_corpus, id2word=id2word, num_topics=K)\n",
    "tfidf_lsi_model.print_topics(num_topics=K, num_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "88124b53-9322-40d4-a8a0-671a54c518ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.096438</td>\n",
       "      <td>-0.064345</td>\n",
       "      <td>0.081752</td>\n",
       "      <td>0.187791</td>\n",
       "      <td>0.037621</td>\n",
       "      <td>-0.006134</td>\n",
       "      <td>0.007457</td>\n",
       "      <td>0.057292</td>\n",
       "      <td>-0.000687</td>\n",
       "      <td>0.014441</td>\n",
       "      <td>0.003018</td>\n",
       "      <td>0.001535</td>\n",
       "      <td>0.011821</td>\n",
       "      <td>-0.005472</td>\n",
       "      <td>0.007129</td>\n",
       "      <td>-0.004340</td>\n",
       "      <td>0.008583</td>\n",
       "      <td>0.028722</td>\n",
       "      <td>0.016516</td>\n",
       "      <td>-0.021231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.121678</td>\n",
       "      <td>0.160559</td>\n",
       "      <td>0.019054</td>\n",
       "      <td>0.029774</td>\n",
       "      <td>-0.163131</td>\n",
       "      <td>0.153800</td>\n",
       "      <td>0.162392</td>\n",
       "      <td>0.086754</td>\n",
       "      <td>0.050644</td>\n",
       "      <td>-0.036742</td>\n",
       "      <td>-0.112626</td>\n",
       "      <td>-0.005560</td>\n",
       "      <td>-0.088801</td>\n",
       "      <td>0.015986</td>\n",
       "      <td>-0.030117</td>\n",
       "      <td>0.060828</td>\n",
       "      <td>-0.065504</td>\n",
       "      <td>0.079354</td>\n",
       "      <td>-0.003550</td>\n",
       "      <td>0.047949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.114348</td>\n",
       "      <td>-0.088036</td>\n",
       "      <td>-0.008067</td>\n",
       "      <td>-0.033921</td>\n",
       "      <td>0.019773</td>\n",
       "      <td>0.029750</td>\n",
       "      <td>-0.024900</td>\n",
       "      <td>0.051668</td>\n",
       "      <td>0.268112</td>\n",
       "      <td>-0.011121</td>\n",
       "      <td>0.014658</td>\n",
       "      <td>0.014457</td>\n",
       "      <td>0.019403</td>\n",
       "      <td>0.043303</td>\n",
       "      <td>0.136160</td>\n",
       "      <td>-0.001862</td>\n",
       "      <td>-0.068640</td>\n",
       "      <td>-0.010558</td>\n",
       "      <td>-0.041912</td>\n",
       "      <td>-0.035772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.099667</td>\n",
       "      <td>0.127528</td>\n",
       "      <td>0.147981</td>\n",
       "      <td>-0.095943</td>\n",
       "      <td>0.058095</td>\n",
       "      <td>0.016658</td>\n",
       "      <td>-0.025935</td>\n",
       "      <td>0.129511</td>\n",
       "      <td>-0.024849</td>\n",
       "      <td>0.042916</td>\n",
       "      <td>-0.037102</td>\n",
       "      <td>-0.096026</td>\n",
       "      <td>-0.203185</td>\n",
       "      <td>0.067233</td>\n",
       "      <td>0.072508</td>\n",
       "      <td>-0.063512</td>\n",
       "      <td>0.133542</td>\n",
       "      <td>0.040367</td>\n",
       "      <td>-0.111224</td>\n",
       "      <td>-0.175234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.115398</td>\n",
       "      <td>0.119539</td>\n",
       "      <td>0.104279</td>\n",
       "      <td>-0.072874</td>\n",
       "      <td>0.050134</td>\n",
       "      <td>-0.046047</td>\n",
       "      <td>-0.020154</td>\n",
       "      <td>0.020553</td>\n",
       "      <td>-0.007696</td>\n",
       "      <td>0.042768</td>\n",
       "      <td>-0.003114</td>\n",
       "      <td>-0.006774</td>\n",
       "      <td>0.023941</td>\n",
       "      <td>-0.006057</td>\n",
       "      <td>-0.021021</td>\n",
       "      <td>0.006679</td>\n",
       "      <td>-0.054390</td>\n",
       "      <td>-0.002607</td>\n",
       "      <td>0.057195</td>\n",
       "      <td>0.030980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18841</th>\n",
       "      <td>0.111611</td>\n",
       "      <td>-0.040149</td>\n",
       "      <td>-0.002239</td>\n",
       "      <td>-0.007048</td>\n",
       "      <td>0.008484</td>\n",
       "      <td>0.007556</td>\n",
       "      <td>-0.009869</td>\n",
       "      <td>-0.029927</td>\n",
       "      <td>-0.010241</td>\n",
       "      <td>-0.021003</td>\n",
       "      <td>-0.027877</td>\n",
       "      <td>-0.046461</td>\n",
       "      <td>0.001771</td>\n",
       "      <td>-0.032864</td>\n",
       "      <td>0.006692</td>\n",
       "      <td>0.007275</td>\n",
       "      <td>0.000921</td>\n",
       "      <td>-0.010309</td>\n",
       "      <td>0.008605</td>\n",
       "      <td>-0.010627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18842</th>\n",
       "      <td>0.041121</td>\n",
       "      <td>0.016629</td>\n",
       "      <td>-0.014593</td>\n",
       "      <td>0.004803</td>\n",
       "      <td>0.011255</td>\n",
       "      <td>0.023225</td>\n",
       "      <td>0.002360</td>\n",
       "      <td>-0.025793</td>\n",
       "      <td>-0.004557</td>\n",
       "      <td>-0.032545</td>\n",
       "      <td>-0.000295</td>\n",
       "      <td>-0.029458</td>\n",
       "      <td>0.028135</td>\n",
       "      <td>0.022948</td>\n",
       "      <td>0.011118</td>\n",
       "      <td>0.030526</td>\n",
       "      <td>0.031692</td>\n",
       "      <td>-0.015422</td>\n",
       "      <td>0.001622</td>\n",
       "      <td>0.006162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18843</th>\n",
       "      <td>0.059817</td>\n",
       "      <td>0.022083</td>\n",
       "      <td>0.036148</td>\n",
       "      <td>-0.018361</td>\n",
       "      <td>-0.019430</td>\n",
       "      <td>0.039212</td>\n",
       "      <td>-0.039218</td>\n",
       "      <td>-0.022285</td>\n",
       "      <td>-0.021465</td>\n",
       "      <td>0.012009</td>\n",
       "      <td>-0.001599</td>\n",
       "      <td>0.014291</td>\n",
       "      <td>0.041423</td>\n",
       "      <td>-0.009175</td>\n",
       "      <td>0.044883</td>\n",
       "      <td>-0.026335</td>\n",
       "      <td>-0.010880</td>\n",
       "      <td>0.001946</td>\n",
       "      <td>-0.017933</td>\n",
       "      <td>-0.007067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18844</th>\n",
       "      <td>0.069566</td>\n",
       "      <td>-0.028832</td>\n",
       "      <td>-0.013698</td>\n",
       "      <td>-0.009136</td>\n",
       "      <td>0.014438</td>\n",
       "      <td>-0.002855</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.002505</td>\n",
       "      <td>-0.028657</td>\n",
       "      <td>-0.026990</td>\n",
       "      <td>0.014333</td>\n",
       "      <td>-0.062280</td>\n",
       "      <td>-0.023114</td>\n",
       "      <td>0.026373</td>\n",
       "      <td>-0.033613</td>\n",
       "      <td>-0.013621</td>\n",
       "      <td>-0.043408</td>\n",
       "      <td>0.003408</td>\n",
       "      <td>0.023973</td>\n",
       "      <td>0.014174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18845</th>\n",
       "      <td>0.126166</td>\n",
       "      <td>-0.006882</td>\n",
       "      <td>0.006112</td>\n",
       "      <td>-0.012644</td>\n",
       "      <td>-0.017865</td>\n",
       "      <td>0.017730</td>\n",
       "      <td>-0.015509</td>\n",
       "      <td>-0.027586</td>\n",
       "      <td>-0.014912</td>\n",
       "      <td>0.010426</td>\n",
       "      <td>-0.003651</td>\n",
       "      <td>-0.005270</td>\n",
       "      <td>0.026437</td>\n",
       "      <td>-0.003909</td>\n",
       "      <td>-0.004550</td>\n",
       "      <td>-0.035527</td>\n",
       "      <td>-0.009218</td>\n",
       "      <td>-0.004775</td>\n",
       "      <td>0.015576</td>\n",
       "      <td>0.000438</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18846 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             0         1         2         3         4         5         6   \\\n",
       "0      0.096438 -0.064345  0.081752  0.187791  0.037621 -0.006134  0.007457   \n",
       "1      0.121678  0.160559  0.019054  0.029774 -0.163131  0.153800  0.162392   \n",
       "2      0.114348 -0.088036 -0.008067 -0.033921  0.019773  0.029750 -0.024900   \n",
       "3      0.099667  0.127528  0.147981 -0.095943  0.058095  0.016658 -0.025935   \n",
       "4      0.115398  0.119539  0.104279 -0.072874  0.050134 -0.046047 -0.020154   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "18841  0.111611 -0.040149 -0.002239 -0.007048  0.008484  0.007556 -0.009869   \n",
       "18842  0.041121  0.016629 -0.014593  0.004803  0.011255  0.023225  0.002360   \n",
       "18843  0.059817  0.022083  0.036148 -0.018361 -0.019430  0.039212 -0.039218   \n",
       "18844  0.069566 -0.028832 -0.013698 -0.009136  0.014438 -0.002855 -0.007736   \n",
       "18845  0.126166 -0.006882  0.006112 -0.012644 -0.017865  0.017730 -0.015509   \n",
       "\n",
       "             7         8         9         10        11        12        13  \\\n",
       "0      0.057292 -0.000687  0.014441  0.003018  0.001535  0.011821 -0.005472   \n",
       "1      0.086754  0.050644 -0.036742 -0.112626 -0.005560 -0.088801  0.015986   \n",
       "2      0.051668  0.268112 -0.011121  0.014658  0.014457  0.019403  0.043303   \n",
       "3      0.129511 -0.024849  0.042916 -0.037102 -0.096026 -0.203185  0.067233   \n",
       "4      0.020553 -0.007696  0.042768 -0.003114 -0.006774  0.023941 -0.006057   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "18841 -0.029927 -0.010241 -0.021003 -0.027877 -0.046461  0.001771 -0.032864   \n",
       "18842 -0.025793 -0.004557 -0.032545 -0.000295 -0.029458  0.028135  0.022948   \n",
       "18843 -0.022285 -0.021465  0.012009 -0.001599  0.014291  0.041423 -0.009175   \n",
       "18844 -0.002505 -0.028657 -0.026990  0.014333 -0.062280 -0.023114  0.026373   \n",
       "18845 -0.027586 -0.014912  0.010426 -0.003651 -0.005270  0.026437 -0.003909   \n",
       "\n",
       "             14        15        16        17        18        19  \n",
       "0      0.007129 -0.004340  0.008583  0.028722  0.016516 -0.021231  \n",
       "1     -0.030117  0.060828 -0.065504  0.079354 -0.003550  0.047949  \n",
       "2      0.136160 -0.001862 -0.068640 -0.010558 -0.041912 -0.035772  \n",
       "3      0.072508 -0.063512  0.133542  0.040367 -0.111224 -0.175234  \n",
       "4     -0.021021  0.006679 -0.054390 -0.002607  0.057195  0.030980  \n",
       "...         ...       ...       ...       ...       ...       ...  \n",
       "18841  0.006692  0.007275  0.000921 -0.010309  0.008605 -0.010627  \n",
       "18842  0.011118  0.030526  0.031692 -0.015422  0.001622  0.006162  \n",
       "18843  0.044883 -0.026335 -0.010880  0.001946 -0.017933 -0.007067  \n",
       "18844 -0.033613 -0.013621 -0.043408  0.003408  0.023973  0.014174  \n",
       "18845 -0.004550 -0.035527 -0.009218 -0.004775  0.015576  0.000438  \n",
       "\n",
       "[18846 rows x 20 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows = []\n",
    "for doc in tfidf_corpus:\n",
    "    doc_top = []\n",
    "    for t in tfidf_lsi_model[doc]:\n",
    "        doc_top.append(t[1])\n",
    "    rows.append(doc_top)\n",
    "\n",
    "document_topic_matrix = pd.DataFrame(rows)\n",
    "document_topic_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "dfb27939-c126-4a6f-a174-95a5b7bdb360",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_topics = 20\n",
    "lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,\n",
    "                                           id2word=id2word,\n",
    "                                           num_topics=num_topics,\n",
    "                                           random_state=100,\n",
    "                                           update_every=1,\n",
    "                                           chunksize=100,\n",
    "                                           passes=10,\n",
    "                                           alpha='auto',\n",
    "                                           per_word_topics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "574450d0-0c6a-46c6-94b1-565546363bf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = []\n",
    "for doc in corpus:\n",
    "    doc_top = []\n",
    "    for t in lda_model.get_document_topics(doc, minimum_probability = 0):\n",
    "        doc_top.append(t[1])\n",
    "    rows.append(doc_top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "60258582-1c1b-404f-b16b-78de19e725a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.001273</td>\n",
       "      <td>0.003925</td>\n",
       "      <td>0.027564</td>\n",
       "      <td>0.003541</td>\n",
       "      <td>0.002531</td>\n",
       "      <td>0.070713</td>\n",
       "      <td>0.001256</td>\n",
       "      <td>0.002608</td>\n",
       "      <td>0.016298</td>\n",
       "      <td>0.016849</td>\n",
       "      <td>0.005205</td>\n",
       "      <td>0.018300</td>\n",
       "      <td>0.014934</td>\n",
       "      <td>0.002469</td>\n",
       "      <td>0.003555</td>\n",
       "      <td>0.534880</td>\n",
       "      <td>0.004282</td>\n",
       "      <td>0.232538</td>\n",
       "      <td>0.034169</td>\n",
       "      <td>0.003112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.001799</td>\n",
       "      <td>0.005543</td>\n",
       "      <td>0.025120</td>\n",
       "      <td>0.005002</td>\n",
       "      <td>0.003574</td>\n",
       "      <td>0.051309</td>\n",
       "      <td>0.001773</td>\n",
       "      <td>0.003684</td>\n",
       "      <td>0.020009</td>\n",
       "      <td>0.257890</td>\n",
       "      <td>0.007352</td>\n",
       "      <td>0.025289</td>\n",
       "      <td>0.021113</td>\n",
       "      <td>0.003487</td>\n",
       "      <td>0.005021</td>\n",
       "      <td>0.048155</td>\n",
       "      <td>0.006048</td>\n",
       "      <td>0.184649</td>\n",
       "      <td>0.318788</td>\n",
       "      <td>0.004396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000760</td>\n",
       "      <td>0.011509</td>\n",
       "      <td>0.213770</td>\n",
       "      <td>0.002209</td>\n",
       "      <td>0.001510</td>\n",
       "      <td>0.120583</td>\n",
       "      <td>0.000749</td>\n",
       "      <td>0.001556</td>\n",
       "      <td>0.129346</td>\n",
       "      <td>0.022957</td>\n",
       "      <td>0.003502</td>\n",
       "      <td>0.035050</td>\n",
       "      <td>0.017883</td>\n",
       "      <td>0.001473</td>\n",
       "      <td>0.002121</td>\n",
       "      <td>0.065833</td>\n",
       "      <td>0.011707</td>\n",
       "      <td>0.324563</td>\n",
       "      <td>0.031060</td>\n",
       "      <td>0.001857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.001104</td>\n",
       "      <td>0.016716</td>\n",
       "      <td>0.002096</td>\n",
       "      <td>0.003069</td>\n",
       "      <td>0.002193</td>\n",
       "      <td>0.055757</td>\n",
       "      <td>0.001088</td>\n",
       "      <td>0.002260</td>\n",
       "      <td>0.025545</td>\n",
       "      <td>0.013851</td>\n",
       "      <td>0.004512</td>\n",
       "      <td>0.029638</td>\n",
       "      <td>0.013067</td>\n",
       "      <td>0.002140</td>\n",
       "      <td>0.003081</td>\n",
       "      <td>0.047890</td>\n",
       "      <td>0.003711</td>\n",
       "      <td>0.165635</td>\n",
       "      <td>0.603948</td>\n",
       "      <td>0.002698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.001505</td>\n",
       "      <td>0.004637</td>\n",
       "      <td>0.002857</td>\n",
       "      <td>0.004184</td>\n",
       "      <td>0.166455</td>\n",
       "      <td>0.053241</td>\n",
       "      <td>0.001484</td>\n",
       "      <td>0.003081</td>\n",
       "      <td>0.104342</td>\n",
       "      <td>0.064048</td>\n",
       "      <td>0.006151</td>\n",
       "      <td>0.060891</td>\n",
       "      <td>0.017756</td>\n",
       "      <td>0.002917</td>\n",
       "      <td>0.004200</td>\n",
       "      <td>0.040514</td>\n",
       "      <td>0.005059</td>\n",
       "      <td>0.260168</td>\n",
       "      <td>0.192832</td>\n",
       "      <td>0.003678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18841</th>\n",
       "      <td>0.000872</td>\n",
       "      <td>0.023726</td>\n",
       "      <td>0.001658</td>\n",
       "      <td>0.065548</td>\n",
       "      <td>0.022764</td>\n",
       "      <td>0.091994</td>\n",
       "      <td>0.000859</td>\n",
       "      <td>0.001785</td>\n",
       "      <td>0.012103</td>\n",
       "      <td>0.010722</td>\n",
       "      <td>0.014081</td>\n",
       "      <td>0.035512</td>\n",
       "      <td>0.031157</td>\n",
       "      <td>0.012189</td>\n",
       "      <td>0.002433</td>\n",
       "      <td>0.075341</td>\n",
       "      <td>0.002930</td>\n",
       "      <td>0.559376</td>\n",
       "      <td>0.032818</td>\n",
       "      <td>0.002130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18842</th>\n",
       "      <td>0.002667</td>\n",
       "      <td>0.040403</td>\n",
       "      <td>0.005065</td>\n",
       "      <td>0.007417</td>\n",
       "      <td>0.005300</td>\n",
       "      <td>0.240163</td>\n",
       "      <td>0.002630</td>\n",
       "      <td>0.005462</td>\n",
       "      <td>0.030797</td>\n",
       "      <td>0.059234</td>\n",
       "      <td>0.010903</td>\n",
       "      <td>0.037362</td>\n",
       "      <td>0.031280</td>\n",
       "      <td>0.005171</td>\n",
       "      <td>0.007446</td>\n",
       "      <td>0.093540</td>\n",
       "      <td>0.008968</td>\n",
       "      <td>0.293085</td>\n",
       "      <td>0.106587</td>\n",
       "      <td>0.006519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18843</th>\n",
       "      <td>0.002068</td>\n",
       "      <td>0.066188</td>\n",
       "      <td>0.003927</td>\n",
       "      <td>0.005751</td>\n",
       "      <td>0.004110</td>\n",
       "      <td>0.129584</td>\n",
       "      <td>0.002039</td>\n",
       "      <td>0.004235</td>\n",
       "      <td>0.022934</td>\n",
       "      <td>0.024913</td>\n",
       "      <td>0.008454</td>\n",
       "      <td>0.053652</td>\n",
       "      <td>0.024253</td>\n",
       "      <td>0.004009</td>\n",
       "      <td>0.005773</td>\n",
       "      <td>0.125619</td>\n",
       "      <td>0.006953</td>\n",
       "      <td>0.299732</td>\n",
       "      <td>0.125863</td>\n",
       "      <td>0.079942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18844</th>\n",
       "      <td>0.002235</td>\n",
       "      <td>0.006890</td>\n",
       "      <td>0.004245</td>\n",
       "      <td>0.006216</td>\n",
       "      <td>0.004442</td>\n",
       "      <td>0.137721</td>\n",
       "      <td>0.002204</td>\n",
       "      <td>0.004578</td>\n",
       "      <td>0.026082</td>\n",
       "      <td>0.027076</td>\n",
       "      <td>0.009138</td>\n",
       "      <td>0.031313</td>\n",
       "      <td>0.080185</td>\n",
       "      <td>0.004334</td>\n",
       "      <td>0.033223</td>\n",
       "      <td>0.113506</td>\n",
       "      <td>0.034463</td>\n",
       "      <td>0.396995</td>\n",
       "      <td>0.069689</td>\n",
       "      <td>0.005464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18845</th>\n",
       "      <td>0.000910</td>\n",
       "      <td>0.002808</td>\n",
       "      <td>0.001728</td>\n",
       "      <td>0.027330</td>\n",
       "      <td>0.012790</td>\n",
       "      <td>0.214556</td>\n",
       "      <td>0.000897</td>\n",
       "      <td>0.001863</td>\n",
       "      <td>0.022345</td>\n",
       "      <td>0.057127</td>\n",
       "      <td>0.025681</td>\n",
       "      <td>0.040987</td>\n",
       "      <td>0.053312</td>\n",
       "      <td>0.001764</td>\n",
       "      <td>0.015153</td>\n",
       "      <td>0.063214</td>\n",
       "      <td>0.003059</td>\n",
       "      <td>0.332911</td>\n",
       "      <td>0.108374</td>\n",
       "      <td>0.013192</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18846 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             0         1         2         3         4         5         6   \\\n",
       "0      0.001273  0.003925  0.027564  0.003541  0.002531  0.070713  0.001256   \n",
       "1      0.001799  0.005543  0.025120  0.005002  0.003574  0.051309  0.001773   \n",
       "2      0.000760  0.011509  0.213770  0.002209  0.001510  0.120583  0.000749   \n",
       "3      0.001104  0.016716  0.002096  0.003069  0.002193  0.055757  0.001088   \n",
       "4      0.001505  0.004637  0.002857  0.004184  0.166455  0.053241  0.001484   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "18841  0.000872  0.023726  0.001658  0.065548  0.022764  0.091994  0.000859   \n",
       "18842  0.002667  0.040403  0.005065  0.007417  0.005300  0.240163  0.002630   \n",
       "18843  0.002068  0.066188  0.003927  0.005751  0.004110  0.129584  0.002039   \n",
       "18844  0.002235  0.006890  0.004245  0.006216  0.004442  0.137721  0.002204   \n",
       "18845  0.000910  0.002808  0.001728  0.027330  0.012790  0.214556  0.000897   \n",
       "\n",
       "             7         8         9         10        11        12        13  \\\n",
       "0      0.002608  0.016298  0.016849  0.005205  0.018300  0.014934  0.002469   \n",
       "1      0.003684  0.020009  0.257890  0.007352  0.025289  0.021113  0.003487   \n",
       "2      0.001556  0.129346  0.022957  0.003502  0.035050  0.017883  0.001473   \n",
       "3      0.002260  0.025545  0.013851  0.004512  0.029638  0.013067  0.002140   \n",
       "4      0.003081  0.104342  0.064048  0.006151  0.060891  0.017756  0.002917   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "18841  0.001785  0.012103  0.010722  0.014081  0.035512  0.031157  0.012189   \n",
       "18842  0.005462  0.030797  0.059234  0.010903  0.037362  0.031280  0.005171   \n",
       "18843  0.004235  0.022934  0.024913  0.008454  0.053652  0.024253  0.004009   \n",
       "18844  0.004578  0.026082  0.027076  0.009138  0.031313  0.080185  0.004334   \n",
       "18845  0.001863  0.022345  0.057127  0.025681  0.040987  0.053312  0.001764   \n",
       "\n",
       "             14        15        16        17        18        19  \n",
       "0      0.003555  0.534880  0.004282  0.232538  0.034169  0.003112  \n",
       "1      0.005021  0.048155  0.006048  0.184649  0.318788  0.004396  \n",
       "2      0.002121  0.065833  0.011707  0.324563  0.031060  0.001857  \n",
       "3      0.003081  0.047890  0.003711  0.165635  0.603948  0.002698  \n",
       "4      0.004200  0.040514  0.005059  0.260168  0.192832  0.003678  \n",
       "...         ...       ...       ...       ...       ...       ...  \n",
       "18841  0.002433  0.075341  0.002930  0.559376  0.032818  0.002130  \n",
       "18842  0.007446  0.093540  0.008968  0.293085  0.106587  0.006519  \n",
       "18843  0.005773  0.125619  0.006953  0.299732  0.125863  0.079942  \n",
       "18844  0.033223  0.113506  0.034463  0.396995  0.069689  0.005464  \n",
       "18845  0.015153  0.063214  0.003059  0.332911  0.108374  0.013192  \n",
       "\n",
       "[18846 rows x 20 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_topic_matrix = pd.DataFrame(rows)\n",
    "#document_topic_matrix_sourcecode[\"identifier\"] = df_sourcecode.iloc[:,0].tolist()\n",
    "document_topic_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "d90885b8-8056-4fca-b77c-b945d9fbf86e",
   "metadata": {},
   "outputs": [],
   "source": [
    "document_topic_matrix_short = document_topic_matrix\n",
    "Y_short = Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a091f281-8365-4461-9b2b-a82d5df00c64",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "819f57cc-3beb-4115-8c98-e82e5ea9dc43",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Daniel Atzberger\\anaconda3\\Lib\\site-packages\\scipy\\spatial\\distance.py:1259: RuntimeWarning: invalid value encountered in sqrt\n",
      "  return np.sqrt(js / 2.0)\n"
     ]
    }
   ],
   "source": [
    "# Dimensionality reduction 1: tSNE\n",
    "\n",
    "import time\n",
    "from sklearn.manifold import TSNE\n",
    "from scipy.spatial.distance import pdist, squareform, jensenshannon, cosine\n",
    "\n",
    "#n_sne = 7000\n",
    "\n",
    "time_start = time.time()\n",
    "#tsne = TSNE(n_iter=300)\n",
    "tsne = TSNE(n_components=2, n_iter = 250, perplexity=30, learning_rate = 250, metric =jensenshannon)\n",
    "tsne_results = tsne.fit_transform(document_topic_matrix_short.values)\n",
    "\n",
    "print ('t-SNE done! Time elapsed: {} seconds'.format(time.time()-time_start))\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "# Create the figure\n",
    "fig = plt.figure( figsize=(8,8) )\n",
    "ax = fig.add_subplot(1, 1, 1, title='TSNE' )\n",
    "# Create the scatter\n",
    "ax.scatter(\n",
    "    x=tsne_results[:,0], \n",
    "    y=tsne_results[:,1], \n",
    "    c=Y_short, \n",
    "    cmap=plt.cm.get_cmap('Paired'), \n",
    "    alpha=0.4)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecc389a8-03aa-4eac-858f-56c0fb46b30c",
   "metadata": {},
   "source": [
    "### Local Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a714db0a-0fb4-49fc-82c2-6b10663a26ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial.distance import cdist\n",
    "\n",
    "def pairwise_distance_matrix(point, distance_function=\"euclidean\"):\n",
    "\t\"\"\"\n",
    "\tCompute the pairwise distance matrix of the point list\n",
    "\tYou can use any distance function from scipy.spatial.distance.cdist or specify a callable function\n",
    "\tINPUT:\n",
    "\t\tndarray: point: list of points\n",
    "\t\tstr or callable: distance_function: distance function to use\n",
    "\tOUTPUT:\n",
    "\t\tndarry: pairwise distance matrix \n",
    "\t\"\"\"\n",
    "\tif callable(distance_function):\n",
    "\t\tdistance_matrix = cdist(point, point, distance_function)\n",
    "\telse:\n",
    "\t\tdistance_matrix = cdist(point, point, distance_function)\n",
    "\treturn distance_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a3d5600-5886-4e97-9796-0fd20efd4abb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from .pairwise_dist import pairwise_distance_matrix\n",
    "import numpy as np\n",
    "import faiss\n",
    "from sklearn.neighbors import KDTree\n",
    "\n",
    "def knn_with_ranking(points, k, distance_function='euclidean'):\n",
    "    \"\"\"\n",
    "    Compute the k-nearest neighbors of the points along with the \n",
    "    rankings of other points based on the distance to each point.\n",
    "    If the distance matrix is not provided, it is computed in O(n^2) time.\n",
    "    INPUT:\n",
    "    \tndarray: points: list of points\n",
    "        int: k: number of nearest neighbors to compute\n",
    "    \tndarray: distance_matrix: pairwise distance matrix (Optional)\n",
    "    OUTPUT:\n",
    "    \tndarray: knn_indices: k-nearest neighbors of each point \n",
    "    \tndarray: ranking: ranking of other points based on the distance to each point\n",
    "    \"\"\"\n",
    "    distance_matrix = pairwise_distance_matrix(points, distance_function)\n",
    "    \n",
    "    knn_indices = np.empty((points.shape[0], k), dtype=np.int32)\n",
    "    ranking = np.empty((points.shape[0], points.shape[0]), dtype=np.int32)\n",
    "      \n",
    "    for i in range(points.shape[0]):\n",
    "        distance_to_i = distance_matrix[i]\n",
    "        sorted_indices = np.argsort(distance_to_i)\n",
    "        knn_indices[i] = sorted_indices[1:k+1]\n",
    "        ranking[i] = np.argsort(sorted_indices)\n",
    "      \n",
    "    return knn_indices, ranking\n",
    "  \n",
    "\n",
    "def knn(points, k, distance_function=\"euclidean\"):\n",
    "    \"\"\"\n",
    "    Compute the k-nearest neighbors of the points\n",
    "    If the distance function is euclidean, the computation relies on faiss-cpu.\n",
    "    Otherwise, the computation is done based on scikit-learn KD Tree algorithm\n",
    "    You can use any distance function supported by scikit-learn KD Tree or specify a callable function\n",
    "    INPUT:\n",
    "    \tndarray: points: list of points\n",
    "    \tint: k: number of nearest neighbors to compute\n",
    "    \tstr or callable: distance_function: distance function to use\n",
    "    OUTPUT:\n",
    "    \tndarray: knn_indices: k-nearest neighbors of each point \n",
    "    \"\"\"\n",
    "    \t\n",
    "    ## make c-contiguous\n",
    "    points = np.ascontiguousarray(points, dtype=np.float32)\n",
    "    \n",
    "    if distance_function == \"euclidean\":\n",
    "        index = faiss.IndexFlatL2(points.shape[1])\n",
    "        index.add(points)\n",
    "        knn_indices = index.search(points, k+1)[1][:, 1:]\n",
    "    else:\n",
    "        tree = KDTree(points, metric=distance_function)\n",
    "        knn_indices = tree.query(points, k=k+1, return_distance=False)[:, 1:]\n",
    "    \t\n",
    "    return knn_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e73615e-5aa9-44ce-a792-5182d0c4f288",
   "metadata": {},
   "outputs": [],
   "source": [
    "def measure(orig, emb, k=20, return_local=False):\n",
    "\t\"\"\"\n",
    "\tCompute the trustworthiness and continuity of the embedding\n",
    "\tINPUT:\n",
    "\t\tndarray: orig: original data\n",
    "\t\tndarray: emb: embedded data\n",
    "\t\tint: k: number of nearest neighbors to consider\n",
    "\t\ttuple: knn_ranking_info: precomputed k-nearest neighbors and rankings of the original and embedded data (Optional)\n",
    "\tOUTPUT:\n",
    "\t\tdict: trustworthiness and continuity\n",
    "\t\"\"\"\n",
    "\n",
    "\torig_knn_indices, orig_ranking = knn_with_ranking(orig, k, distance_function='cosine')\n",
    "\temb_knn_indices,  emb_ranking  = knn_with_ranking(emb, k)\n",
    "\n",
    "\tif return_local:\n",
    "\t\ttrust, local_trust = tnc_computation(orig_knn_indices, orig_ranking, emb_knn_indices, k, return_local)\n",
    "\t\tcont , local_cont  = tnc_computation(emb_knn_indices,  emb_ranking, orig_knn_indices, k, return_local)\n",
    "\t\treturn ({\n",
    "\t\t\t\"trustworthiness\": trust,\n",
    "\t\t\t\"continuity\": cont\n",
    "\t\t}, {\n",
    "\t\t\t\"local_trustworthiness\": local_trust,\n",
    "\t\t\t\"local_continuity\": local_cont\n",
    "\t\t})\n",
    "\telse:\n",
    "\t\ttrust = tnc_computation(orig_knn_indices, orig_ranking, emb_knn_indices, k, return_local)\n",
    "\t\tcont  = tnc_computation(emb_knn_indices,  emb_ranking, orig_knn_indices, k, return_local)\n",
    "\t\treturn {\n",
    "\t\t\t\"trustworthiness\": trust,\n",
    "\t\t\t\"continuity\": cont\n",
    "\t\t}\n",
    "\n",
    "def tnc_computation(base_knn_indices, base_ranking, target_knn_indices, k, return_local=False):\n",
    "\t\"\"\"\n",
    "\tCore computation of trustworthiness and continuity\n",
    "\t\"\"\"\n",
    "\tlocal_distortion_list = []\n",
    "\tpoints_num = base_knn_indices.shape[0]\n",
    "\n",
    "\tfor i in range(points_num):\n",
    "\t\tmissings = np.setdiff1d(target_knn_indices[i], base_knn_indices[i])\n",
    "\t\tlocal_distortion = 0.0 \n",
    "\t\tfor missing in missings:\n",
    "\t\t\tlocal_distortion += base_ranking[i, missing] - k\n",
    "\t\tlocal_distortion_list.append(local_distortion)\n",
    "\tlocal_distortion_list = np.array(local_distortion_list)\n",
    "\tlocal_distortion_list = 1 - local_distortion_list * (2 / (k * (2 * points_num - 3 * k - 1)))\n",
    "\n",
    "\taverage_distortion = np.mean(local_distortion_list)\n",
    "\n",
    "\tif return_local:\n",
    "\t\treturn average_distortion, local_distortion_list\n",
    "\telse:\n",
    "\t\treturn average_distortion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc68020b-6b0f-40b8-aa98-5319e973d0cf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd31cc4d-b6a9-44be-a76a-434b7527ea76",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c46a57bc-1a05-4af3-a6a0-2b6679fd2e99",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3279628f-f648-4b40-b9bc-321ae736f643",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e239b8d6-f055-46e7-a213-3a5f55a1c725",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}