feat: topic tren analysis

ahmaddynugroho · Jun 30, 2023 · 55d0508 · 55d0508
1 parent 6c8462e
commit 55d0508
Show file tree

Hide file tree

Showing 2 changed files with 371 additions and 0 deletions.
diff --git a/notebook/tren.ipynb b/notebook/tren.ipynb
diff --git a/notebook/tren.py b/notebook/tren.py
@@ -0,0 +1,21 @@
+from pickle import load
+import pandas as pd
+
+print('[INFO] preparing data')
+df = pd.read_parquet('../dist/hp.parquet')
+
+with open('../dist/dictionary.pickle', 'rb') as f:
+    dictionary = load(f)
+with open('../dist/corpus.pickle', 'rb') as f:
+    corpus = load(f)
+with open('../dist/model.pickle', 'rb') as f:
+    model = load(f)
+
+print('[INFO] building corpus')
+df['corpus'] = corpus
+
+print('[INFO] building topics')
+df['topics'] = model.get_document_topics(df['corpus'])
+
+print('[INFO] saving df')
+df.to_parquet('../dist/hpct.parquet')