LDA_perplexity.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "3ffa5b9ea856ed8ffaf5105c7cda503ceb4538c7650d33f63eca5c51fb008d20"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import CoherenceModel, LdaModel\n",
    "from gensim.corpora import Dictionary\n",
    "from gensim import corpora\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "c:\\Users\\pc127\\project2\\Retrieval_of_Candidate_Documents\n"
     ]
    }
   ],
   "source": [
    "import os \n",
    "print( os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda  = LdaModel.load('.../data/LDA_data/2021032315_Step2_clean_data_5/model5.gensim')\n",
    "import pickle\n",
    "with open(\".../data/LDA_data/2021032315_Step2_clean_data_5/corpus.pkl\",'rb') as f:\n",
    "    corpus = pickle.load(f)\n",
    "dictionary = Dictionary.load(\".../data/LDA_data/2021032315_Step2_clean_data_5/dictionary.gensim\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "TypeError",
     "evalue": "log_perplexity() missing 1 required positional argument: 'chunk'",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-ccb2d8e3898e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mPerplexity\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mLdaModel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_perplexity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlda\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: log_perplexity() missing 1 required positional argument: 'chunk'"
     ]
    }
   ],
   "source": [
    "Perplexity=LdaModel.log_perplexity(lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import LDA\n",
    "# data = Load_data('Step2_clean_data.pkl')\n",
    "# path = create_path(args.top_num,args.doc)\n",
    "# dictionary,corpus,ldamodel = Train_LDA(path,doc,args.top_num,args.seed)\n",
    "# LDA.Train_LDA()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    " with open(\"foo.txt\", \"a\") as f:\n",
    "     f.write(\"new ffff\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "'Topic number 95  is finished and use time 3766.8861157894135'"
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3319, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-11-6d188825b775>\", line 9, in <module>\n",
      "    dictionary,corpus,ldamodel = LDA.Train_LDA(path,data,top_num,seed)\n",
      "  File \"c:\\Users\\pc127\\project2\\Retrieval_of_Candidate_Documents\\LDA.py\", line 39, in Train_LDA\n",
      "    ldamodel = gensim.models.ldamodel.LdaModel(\\\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\gensim\\models\\ldamodel.py\", line 519, in __init__\n",
      "    self.update(corpus, chunks_as_numpy=use_numpy)\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\gensim\\models\\ldamodel.py\", line 965, in update\n",
      "    self.log_perplexity(chunk, total_docs=lencorpus)\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\gensim\\models\\ldamodel.py\", line 821, in log_perplexity\n",
      "    perwordbound = self.bound(chunk, subsample_ratio=subsample_ratio) / (subsample_ratio * corpus_words)\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\gensim\\models\\ldamodel.py\", line 1096, in bound\n",
      "    score += sum(cnt * logsumexp(Elogthetad + Elogbeta[:, int(id)]) for id, cnt in doc)\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\gensim\\models\\ldamodel.py\", line 1096, in <genexpr>\n",
      "    score += sum(cnt * logsumexp(Elogthetad + Elogbeta[:, int(id)]) for id, cnt in doc)\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2034, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 1101, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 319, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 353, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\inspect.py\", line 1503, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\inspect.py\", line 1461, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\inspect.py\", line 708, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"C:\\Users\\pc127\\anaconda3\\lib\\inspect.py\", line 746, in getmodule\n",
      "    f = module.__file__\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m"
     ]
    }
   ],
   "source": [
    "import time\n",
    "data_name = 'Step2_clean_data.pkl'\n",
    "data = LDA.Load_data('Step2_clean_data.pkl')\n",
    "seed = 12345\n",
    "print('start complexity ')\n",
    "for top_num in range(5,105,5)\n",
    "    start  = time.time()\n",
    "    path = LDA.create_path(top_num,data_name)\n",
    "    dictionary,corpus,ldamodel = LDA.Train_LDA(path,data,top_num,seed)\n",
    "    model_Umass = CoherenceModel(model=ldamodel, corpus=corpus, dictionary=dictionary, coherence='u_mass')\n",
    "    model_c_v = CoherenceModel(model=ldamodel, texts=data, dictionary=dictionary,  coherence='c_v')\n",
    "    u = model_Umass.get_coherence()\n",
    "    cv = model_c_v.get_coherence()\n",
    "    clear_output(wait=True)\n",
    "    display(f'Topic number {top_num}  is finished and use time {time.time() -start}')\n",
    "    with open(\"u_mass.txt\", \"a\") as f:\n",
    "        f.write(f\"{u}\\n\")\n",
    "    with open(\"c_v.txt\", \"a\") as f:\n",
    "        f.write(f\"{cv}\\n\")\n",
    "\n",
    "# goodcm = CoherenceModel(model=lda, corpus=corpus, dictionary=dictionary, coherence='u_mass')\n",
    "# badcm = CoherenceModel(model=badLdaModel, corpus=corpus, dictionary=dictionary, coherence='u_mass')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "-0.20560343834639933\n"
     ]
    }
   ],
   "source": [
    "print(goodcm.get_coherence())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}