01.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "20JUN21.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO04ndiIV5/LC1Ru5waw5Df",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/The-Gupta/Time/blob/main/01.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kz_A59-oqFV8"
      },
      "source": [
        "# 20JUN21, SUN\n",
        "# [Public] Location:  https://drive.google.com/drive/folders/1fFoFxY4i3YU-KNkz84_AG9Nftxr4Alnb"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3tHOP6aBqJQG",
        "outputId": "b9dd1749-1fc7-4475-d030-8c5ead8910ef"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4HUYwKvfEuQb"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 894
        },
        "id": "nONIWmjoEa9n",
        "outputId": "451fac69-debb-4830-a079-3f3ecc52c5ef"
      },
      "source": [
        "!pip install pyLDAvis"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyLDAvis\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/03/a5/15a0da6b0150b8b68610cc78af80364a80a9a4c8b6dd5ee549b8989d4b60/pyLDAvis-3.3.1.tar.gz (1.7MB)\n",
            "\u001b[K     |████████████████████████████████| 1.7MB 4.0MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (1.0.1)\n",
            "Collecting numpy>=1.20.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a5/42/560d269f604d3e186a57c21a363e77e199358d054884e61b73e405dd217c/numpy-1.20.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.3MB)\n",
            "\u001b[K     |████████████████████████████████| 15.3MB 187kB/s \n",
            "\u001b[?25hRequirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (2.11.3)\n",
            "Requirement already satisfied: gensim in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (3.6.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (57.0.0)\n",
            "Requirement already satisfied: numexpr in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (2.7.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (0.16.0)\n",
            "Collecting pandas>=1.2.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/51/51/48f3fc47c4e2144da2806dfb6629c4dd1fa3d5a143f9652b141e979a8ca9/pandas-1.2.4-cp37-cp37m-manylinux1_x86_64.whl (9.9MB)\n",
            "\u001b[K     |████████████████████████████████| 9.9MB 177kB/s \n",
            "\u001b[?25hRequirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (0.22.2.post1)\n",
            "Collecting funcy\n",
            "  Downloading https://files.pythonhosted.org/packages/44/52/5cf7401456a461e4b481650dfb8279bc000f31a011d0918904f86e755947/funcy-1.16-py2.py3-none-any.whl\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (0.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (1.4.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->pyLDAvis) (2.0.1)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim->pyLDAvis) (5.1.0)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from gensim->pyLDAvis) (1.15.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.2.0->pyLDAvis) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.2.0->pyLDAvis) (2.8.1)\n",
            "Building wheels for collected packages: pyLDAvis\n",
            "  Building wheel for pyLDAvis (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyLDAvis: filename=pyLDAvis-3.3.1-cp37-none-any.whl size=136897 sha256=c7688d48eb669ad22bba4cc5a0b784c560b29334f6812bce683b9780aac4301c\n",
            "  Stored in directory: /root/.cache/pip/wheels/a0/9c/fc/c6e00689d35c82cf96a8adc70edfe7ba7904374fdac3240ac2\n",
            "Successfully built pyLDAvis\n",
            "\u001b[31mERROR: tensorflow 2.5.0 has requirement numpy~=1.19.2, but you'll have numpy 1.20.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: google-colab 1.0.0 has requirement pandas~=1.1.0; python_version >= \"3.0\", but you'll have pandas 1.2.4 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Installing collected packages: numpy, pandas, funcy, pyLDAvis\n",
            "  Found existing installation: numpy 1.19.5\n",
            "    Uninstalling numpy-1.19.5:\n",
            "      Successfully uninstalled numpy-1.19.5\n",
            "  Found existing installation: pandas 1.1.5\n",
            "    Uninstalling pandas-1.1.5:\n",
            "      Successfully uninstalled pandas-1.1.5\n",
            "Successfully installed funcy-1.16 numpy-1.20.3 pandas-1.2.4 pyLDAvis-3.3.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "numpy",
                  "pandas"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6GiyflCaEw1D"
      },
      "source": [
        "Click &nbsp; 'RESTART RUNTIME', &nbsp; and then &nbsp; 'Yes'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GL_LF6LuqJSD",
        "outputId": "0680596f-bb36-4cf9-90a4-8afdc2272820"
      },
      "source": [
        "cd \"gdrive/MyDrive/Time Magazine\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/gdrive/MyDrive/Time Magazine\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WwQVPbeUqJWa"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_l8H4fDlw3pW",
        "outputId": "ec7fe15c-aa45-4acd-d4e6-8966d6ab2832"
      },
      "source": [
        "import re, os, ast, time, requests, urllib.request\n",
        "from datetime import datetime\n",
        "from pytz import timezone\n",
        "\n",
        "\n",
        "!sudo apt install  tesseract-ocr\n",
        "!pip install pytesseract\n",
        "\n",
        "import pytesseract\n",
        "pytesseract.pytesseract.tesseract_cmd = (r'/usr/bin/tesseract')\n",
        "\n",
        "try:\n",
        "    from PIL import Image\n",
        "except ImportError:\n",
        "    import Image\n",
        "import pytesseract, cv2, os\n",
        "import cv2, random, itertools\n",
        "from google.colab.patches import cv2_imshow\n",
        "import numpy as np\n",
        "from sklearn.cluster import DBSCAN\n",
        "\n",
        "\n",
        "!python -m spacy download en_core_web_sm\n",
        "\n",
        "import spacy\n",
        "nlp = spacy.load('en_core_web_sm')\n",
        "\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "nltk.download('stopwords')\n",
        "\n",
        "stopword_set = set(stopwords.words('english'))\n",
        "\n",
        "\n",
        "from __future__ import print_function\n",
        "\n",
        "import pyLDAvis\n",
        "import pyLDAvis.sklearn\n",
        "pyLDAvis.enable_notebook()\n",
        "\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
        "from sklearn.decomposition import LatentDirichletAllocation"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  tesseract-ocr-eng tesseract-ocr-osd\n",
            "The following NEW packages will be installed:\n",
            "  tesseract-ocr tesseract-ocr-eng tesseract-ocr-osd\n",
            "0 upgraded, 3 newly installed, 0 to remove and 39 not upgraded.\n",
            "Need to get 4,795 kB of archives.\n",
            "After this operation, 15.8 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tesseract-ocr-eng all 4.00~git24-0e00fe6-1.2 [1,588 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tesseract-ocr-osd all 4.00~git24-0e00fe6-1.2 [2,989 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tesseract-ocr amd64 4.00~git2288-10f4998a-2 [218 kB]\n",
            "Fetched 4,795 kB in 1s (4,461 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 3.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package tesseract-ocr-eng.\n",
            "(Reading database ... 160772 files and directories currently installed.)\n",
            "Preparing to unpack .../tesseract-ocr-eng_4.00~git24-0e00fe6-1.2_all.deb ...\n",
            "Unpacking tesseract-ocr-eng (4.00~git24-0e00fe6-1.2) ...\n",
            "Selecting previously unselected package tesseract-ocr-osd.\n",
            "Preparing to unpack .../tesseract-ocr-osd_4.00~git24-0e00fe6-1.2_all.deb ...\n",
            "Unpacking tesseract-ocr-osd (4.00~git24-0e00fe6-1.2) ...\n",
            "Selecting previously unselected package tesseract-ocr.\n",
            "Preparing to unpack .../tesseract-ocr_4.00~git2288-10f4998a-2_amd64.deb ...\n",
            "Unpacking tesseract-ocr (4.00~git2288-10f4998a-2) ...\n",
            "Setting up tesseract-ocr-osd (4.00~git24-0e00fe6-1.2) ...\n",
            "Setting up tesseract-ocr-eng (4.00~git24-0e00fe6-1.2) ...\n",
            "Setting up tesseract-ocr (4.00~git2288-10f4998a-2) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Collecting pytesseract\n",
            "  Downloading https://files.pythonhosted.org/packages/a0/e6/a4e9fc8a93c1318540e8de6d8d4beb5749b7960388a7c7f27799fc2dd016/pytesseract-0.3.7.tar.gz\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from pytesseract) (7.1.2)\n",
            "Building wheels for collected packages: pytesseract\n",
            "  Building wheel for pytesseract (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pytesseract: filename=pytesseract-0.3.7-py2.py3-none-any.whl size=13953 sha256=9c7cba4c3f03d19df33d24441a9f1078aeacff1c147a5eb2be66bdce5005dd37\n",
            "  Stored in directory: /root/.cache/pip/wheels/81/20/7e/1dd0daad1575d5260916bb1e9781246430647adaef4b3ca3b3\n",
            "Successfully built pytesseract\n",
            "Installing collected packages: pytesseract\n",
            "Successfully installed pytesseract-0.3.7\n",
            "Requirement already satisfied: en_core_web_sm==2.2.5 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz#egg=en_core_web_sm==2.2.5 in /usr/local/lib/python3.7/dist-packages (2.2.5)\n",
            "Requirement already satisfied: spacy>=2.2.2 in /usr/local/lib/python3.7/dist-packages (from en_core_web_sm==2.2.5) (2.2.4)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.1.3)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (2.23.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.5)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.0)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (0.4.1)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.20.3)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (2.0.5)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (0.8.2)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.5)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (3.0.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (57.0.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (4.41.1)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (7.4.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (2021.5.30)\n",
            "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (4.5.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.4.1)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_sm')\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/past/types/oldstr.py:5: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working\n",
            "  from collections import Iterable\n",
            "/usr/local/lib/python3.7/dist-packages/past/builtins/misc.py:4: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working\n",
            "  from collections import Mapping\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ooFrIFrxEKuY"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6yOCc6-8rGWk"
      },
      "source": [
        "# Scraping [Time Magazine](https://time.com/vault/)\n",
        "1923 - 2015"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R8SQnOciqJYV"
      },
      "source": [
        "def get_issue_dates(year):\n",
        "    url = 'https://time.com/vault/year/{}/'.format(year)\n",
        "    page = requests.get(url)\n",
        "    issue_dates  =  re.findall('\\\"coverDate\\\":\\\"(\\d\\d\\d\\d-\\d\\d-\\d\\d)\\\"', page.text)\n",
        "    return issue_dates\n",
        "\n",
        "def download_page_image(url, path):\n",
        "    try:\n",
        "        urllib.request.urlretrieve(url, path)\n",
        "        return True\n",
        "    except Exception as e:\n",
        "        print('\\n\\tURL:', url, '\\n\\tPath:', path)\n",
        "        print('\\t', e)\n",
        "        return False\n",
        "\n",
        "def download_issue(issue_date):\n",
        "    year, month, day  =  issue_date.split('-')\n",
        "    \n",
        "    # Find Number of Pages\n",
        "    url = 'https://time.com/vault/issue/{}/page/1/'.format(issue_date)\n",
        "    page = requests.get(url)\n",
        "    page_count = len(ast.literal_eval(re.findall('\\\"pagePDFs\\\":(\\[[^\\[\\]]*\\])', page.text)[0]))\n",
        "\n",
        "    # Create Issue Directory\n",
        "    if not os.path.exists(year+'/'+issue_date):    os.mkdir(year+'/'+issue_date)\n",
        "\n",
        "    # Download Pages\n",
        "    for  page_number  in range(1, page_count+1):\n",
        "        page_url  =  'https://content.time.com/time/subscriber/vault/{}/{}/{}/{}/1550.jpg'.format(year, month, year+month+day, page_number)\n",
        "        \n",
        "        # urllib.request.urlretrieve(page_url, '{}/{}/{}.jpg'.format(year, issue_date, page_number))\n",
        "        count = 0\n",
        "        success  =  download_page_image(page_url, '{}/{}/{}.jpg'.format(year, issue_date, page_number))\n",
        "        while not success and count < 3:\n",
        "            success  =  download_page_image(page_url, '{}/{}/{}.jpg'.format(year, issue_date, page_number))\n",
        "            time.sleep(1)\n",
        "            count += 1\n",
        "\n",
        "\n",
        "def get_timestamp():\n",
        "    now_utc = datetime.now(timezone('UTC'))\n",
        "    return now_utc.astimezone(timezone('Asia/Kolkata')).strftime(\"[%d-%m-%Y %H:%M:%S]\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V71Hxn3YqJbD"
      },
      "source": [
        "%%time\n",
        "\n",
        "\n",
        "for  year  in list(range(1923, 2021+1)):\n",
        "    year = str(year)\n",
        "    \n",
        "    # Create year Directory\n",
        "    if not os.path.exists(year):    os.mkdir(year)\n",
        "\n",
        "    issue_dates  =  get_issue_dates(year)\n",
        "    for  issue_date  in issue_dates:\n",
        "        download_issue(issue_date)\n",
        "\n",
        "\n",
        "    print('\\n{}  Completed: {}.'.format(get_timestamp(), year))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vjNl8HOCqJeE"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B9dlKXepr5P0"
      },
      "source": [
        "# Text Extracation (2000-2015)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uk4d_EmkqJje",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "31512116-016e-4dae-f905-6123a5f4b3fb"
      },
      "source": [
        "for  year  in  range(2000, 2016):\n",
        "    year = str(year)\n",
        "    \n",
        "    for  path,subdirs,files  in os.walk(year):\n",
        "        dir_files = os.listdir(path)\n",
        "        print(path)\n",
        "        \n",
        "        for name in files:\n",
        "            if name.endswith('.jpg')  and  name[:-4]+'.txt' not in dir_files:\n",
        "                img_path = os.path.join(path, name)\n",
        "\n",
        "                image = cv2.imread(img_path)\n",
        "                gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)    # Convert to Black and White\n",
        "                gray = cv2.resize(gray, None, fx = 3, fy = 3, interpolation = cv2.INTER_CUBIC)    # Zoom 3X\n",
        "\n",
        "                text  =  pytesseract.image_to_string(gray)    # OCR\n",
        "\n",
        "                # Save text\n",
        "                f = open(img_path[:-4]+'.txt', 'w')\n",
        "                f.write(text)\n",
        "                f.close()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2000\n",
            "2000/2000-01-01\n",
            "2000/2000-01-17\n",
            "2000/2000-01-24\n",
            "2000/2000-01-31\n",
            "2000/2000-02-07\n",
            "2000/2000-02-14\n",
            "2000/2000-02-21\n",
            "2000/2000-02-28\n",
            "2000/2000-03-06\n",
            "2000/2000-03-13\n",
            "2000/2000-03-20\n",
            "2000/2000-03-27\n",
            "2000/2000-04-03\n",
            "2000/2000-04-10\n",
            "2000/2000-04-17\n",
            "2000/2000-04-24\n",
            "2000/2000-05-01\n",
            "2000/2000-05-08\n",
            "2000/2000-05-15\n",
            "2000/2000-05-22\n",
            "2000/2000-05-29\n",
            "2000/2000-06-05\n",
            "2000/2000-06-12\n",
            "2000/2000-06-19\n",
            "2000/2000-06-26\n",
            "2000/2000-07-03\n",
            "2000/2000-07-10\n",
            "2000/2000-07-17\n",
            "2000/2000-07-24\n",
            "2000/2000-07-31\n",
            "2000/2000-08-07\n",
            "2000/2000-08-14\n",
            "2000/2000-08-21\n",
            "2000/2000-08-28\n",
            "2000/2000-09-04\n",
            "2000/2000-09-11\n",
            "2000/2000-09-18\n",
            "2000/2000-09-25\n",
            "2000/2000-10-02\n",
            "2000/2000-10-09\n",
            "2000/2000-10-16\n",
            "2000/2000-10-23\n",
            "2000/2000-10-30\n",
            "2000/2000-11-06\n",
            "2000/2000-11-13\n",
            "2000/2000-11-20\n",
            "2000/2000-11-27\n",
            "2000/2000-12-04\n",
            "2000/2000-12-11\n",
            "2000/2000-12-18\n",
            "2000/2000-12-25\n",
            "2001\n",
            "2001/2001-01-08\n",
            "2001/2001-01-15\n",
            "2001/2001-01-22\n",
            "2001/2001-01-29\n",
            "2001/2001-02-05\n",
            "2001/2001-02-12\n",
            "2001/2001-02-19\n",
            "2001/2001-02-26\n",
            "2001/2001-03-05\n",
            "2001/2001-03-12\n",
            "2001/2001-03-19\n",
            "2001/2001-03-26\n",
            "2001/2001-04-02\n",
            "2001/2001-04-09\n",
            "2001/2001-04-16\n",
            "2001/2001-04-23\n",
            "2001/2001-04-30\n",
            "2001/2001-05-07\n",
            "2001/2001-05-14\n",
            "2001/2001-05-21\n",
            "2001/2001-05-28\n",
            "2001/2001-06-04\n",
            "2001/2001-06-11\n",
            "2001/2001-06-18\n",
            "2001/2001-06-25\n",
            "2001/2001-07-02\n",
            "2001/2001-07-09\n",
            "2001/2001-07-16\n",
            "2001/2001-07-23\n",
            "2001/2001-07-30\n",
            "2001/2001-08-06\n",
            "2001/2001-08-20\n",
            "2001/2001-08-27\n",
            "2001/2001-09-03\n",
            "2001/2001-09-10\n",
            "2001/2001-09-17\n",
            "2001/2001-09-24\n",
            "2001/2001-10-01\n",
            "2001/2001-10-08\n",
            "2001/2001-10-15\n",
            "2001/2001-10-22\n",
            "2001/2001-10-29\n",
            "2001/2001-11-05\n",
            "2001/2001-11-12\n",
            "2001/2001-11-19\n",
            "2001/2001-11-26\n",
            "2001/2001-12-03\n",
            "2001/2001-12-10\n",
            "2001/2001-12-17\n",
            "2001/2001-12-24\n",
            "2001/2001-12-31\n",
            "2002\n",
            "2002/2002-01-14\n",
            "2002/2002-01-21\n",
            "2002/2002-01-28\n",
            "2002/2002-02-04\n",
            "2002/2002-02-11\n",
            "2002/2002-02-18\n",
            "2002/2002-02-25\n",
            "2002/2002-03-04\n",
            "2002/2002-03-11\n",
            "2002/2002-03-18\n",
            "2002/2002-03-25\n",
            "2002/2002-04-01\n",
            "2002/2002-04-08\n",
            "2002/2002-04-15\n",
            "2002/2002-04-22\n",
            "2002/2002-04-29\n",
            "2002/2002-05-06\n",
            "2002/2002-05-13\n",
            "2002/2002-05-20\n",
            "2002/2002-05-27\n",
            "2002/2002-06-03\n",
            "2002/2002-06-10\n",
            "2002/2002-06-17\n",
            "2002/2002-06-24\n",
            "2002/2002-07-01\n",
            "2002/2002-07-08\n",
            "2002/2002-07-15\n",
            "2002/2002-07-22\n",
            "2002/2002-07-29\n",
            "2002/2002-08-05\n",
            "2002/2002-08-12\n",
            "2002/2002-08-19\n",
            "2002/2002-08-26\n",
            "2002/2002-09-02\n",
            "2002/2002-09-09\n",
            "2002/2002-09-16\n",
            "2002/2002-09-23\n",
            "2002/2002-09-30\n",
            "2002/2002-10-07\n",
            "2002/2002-10-14\n",
            "2002/2002-10-21\n",
            "2002/2002-10-28\n",
            "2002/2002-11-04\n",
            "2002/2002-11-11\n",
            "2002/2002-11-18\n",
            "2002/2002-11-25\n",
            "2002/2002-12-02\n",
            "2002/2002-12-09\n",
            "2002/2002-12-16\n",
            "2002/2002-12-18\n",
            "2002/2002-12-23\n",
            "2002/2002-12-30\n",
            "2003\n",
            "2003/2003-01-13\n",
            "2003/2003-01-20\n",
            "2003/2003-01-27\n",
            "2003/2003-02-03\n",
            "2003/2003-02-10\n",
            "2003/2003-02-17\n",
            "2003/2003-02-24\n",
            "2003/2003-03-03\n",
            "2003/2003-03-10\n",
            "2003/2003-03-17\n",
            "2003/2003-03-24\n",
            "2003/2003-03-31\n",
            "2003/2003-04-07\n",
            "2003/2003-04-14\n",
            "2003/2003-04-21\n",
            "2003/2003-04-28\n",
            "2003/2003-05-05\n",
            "2003/2003-05-12\n",
            "2003/2003-05-19\n",
            "2003/2003-05-26\n",
            "2003/2003-06-02\n",
            "2003/2003-06-09\n",
            "2003/2003-06-16\n",
            "2003/2003-06-23\n",
            "2003/2003-06-30\n",
            "2003/2003-07-07\n",
            "2003/2003-07-14\n",
            "2003/2003-07-21\n",
            "2003/2003-07-28\n",
            "2003/2003-08-04\n",
            "2003/2003-08-11\n",
            "2003/2003-08-18\n",
            "2003/2003-08-25\n",
            "2003/2003-09-01\n",
            "2003/2003-09-08\n",
            "2003/2003-09-15\n",
            "2003/2003-09-22\n",
            "2003/2003-09-29\n",
            "2003/2003-10-06\n",
            "2003/2003-10-13\n",
            "2003/2003-10-20\n",
            "2003/2003-10-27\n",
            "2003/2003-11-03\n",
            "2003/2003-11-10\n",
            "2003/2003-11-24\n",
            "2003/2003-12-01\n",
            "2003/2003-12-08\n",
            "2003/2003-12-15\n",
            "2003/2003-12-22\n",
            "2004\n",
            "2004/2004-01-12\n",
            "2004/2004-01-19\n",
            "2004/2004-01-26\n",
            "2004/2004-02-02\n",
            "2004/2004-02-09\n",
            "2004/2004-02-16\n",
            "2004/2004-02-23\n",
            "2004/2004-03-01\n",
            "2004/2004-03-08\n",
            "2004/2004-03-15\n",
            "2004/2004-03-22\n",
            "2004/2004-03-29\n",
            "2004/2004-04-05\n",
            "2004/2004-04-12\n",
            "2004/2004-04-19\n",
            "2004/2004-04-26\n",
            "2004/2004-05-03\n",
            "2004/2004-05-10\n",
            "2004/2004-05-17\n",
            "2004/2004-05-24\n",
            "2004/2004-05-31\n",
            "2004/2004-06-14\n",
            "2004/2004-06-21\n",
            "2004/2004-06-28\n",
            "2004/2004-07-05\n",
            "2004/2004-07-12\n",
            "2004/2004-07-19\n",
            "2004/2004-07-26\n",
            "2004/2004-08-02\n",
            "2004/2004-08-09\n",
            "2004/2004-08-16\n",
            "2004/2004-08-23\n",
            "2004/2004-08-30\n",
            "2004/2004-09-06\n",
            "2004/2004-09-13\n",
            "2004/2004-09-20\n",
            "2004/2004-09-27\n",
            "2004/2004-10-11\n",
            "2004/2004-10-18\n",
            "2004/2004-10-25\n",
            "2004/2004-11-01\n",
            "2004/2004-11-08\n",
            "2004/2004-11-15\n",
            "2004/2004-11-29\n",
            "2004/2004-12-06\n",
            "2004/2004-12-13\n",
            "2004/2004-12-20\n",
            "2005\n",
            "2005/2005-01-10\n",
            "2005/2005-01-17\n",
            "2005/2005-01-24\n",
            "2005/2005-01-31\n",
            "2005/2005-02-14\n",
            "2005/2005-02-21\n",
            "2005/2005-02-28\n",
            "2005/2005-03-07\n",
            "2005/2005-03-14\n",
            "2005/2005-03-21\n",
            "2005/2005-03-28\n",
            "2005/2005-04-04\n",
            "2005/2005-04-11\n",
            "2005/2005-04-25\n",
            "2005/2005-05-02\n",
            "2005/2005-05-09\n",
            "2005/2005-05-16\n",
            "2005/2005-05-23\n",
            "2005/2005-05-30\n",
            "2005/2005-06-06\n",
            "2005/2005-06-13\n",
            "2005/2005-06-20\n",
            "2005/2005-06-27\n",
            "2005/2005-07-04\n",
            "2005/2005-07-11\n",
            "2005/2005-07-18\n",
            "2005/2005-07-25\n",
            "2005/2005-08-01\n",
            "2005/2005-08-15\n",
            "2005/2005-08-22\n",
            "2005/2005-08-29\n",
            "2005/2005-09-05\n",
            "2005/2005-09-12\n",
            "2005/2005-09-19\n",
            "2005/2005-09-26\n",
            "2005/2005-10-03\n",
            "2005/2005-10-10\n",
            "2005/2005-10-17\n",
            "2005/2005-10-24\n",
            "2005/2005-10-31\n",
            "2005/2005-11-07\n",
            "2005/2005-11-14\n",
            "2005/2005-11-21\n",
            "2005/2005-12-05\n",
            "2005/2005-12-12\n",
            "2005/2005-12-26\n",
            "2006\n",
            "2006/2006-01-16\n",
            "2006/2006-01-23\n",
            "2006/2006-01-30\n",
            "2006/2006-02-06\n",
            "2006/2006-02-13\n",
            "2006/2006-02-20\n",
            "2006/2006-02-27\n",
            "2006/2006-03-06\n",
            "2006/2006-03-20\n",
            "2006/2006-03-27\n",
            "2006/2006-04-03\n",
            "2006/2006-04-10\n",
            "2006/2006-04-17\n",
            "2006/2006-04-24\n",
            "2006/2006-05-01\n",
            "2006/2006-05-08\n",
            "2006/2006-05-15\n",
            "2006/2006-05-22\n",
            "2006/2006-05-29\n",
            "2006/2006-06-05\n",
            "2006/2006-06-12\n",
            "2006/2006-06-19\n",
            "2006/2006-06-26\n",
            "2006/2006-07-03\n",
            "2006/2006-07-10\n",
            "2006/2006-07-17\n",
            "2006/2006-07-24\n",
            "2006/2006-07-31\n",
            "2006/2006-08-07\n",
            "2006/2006-08-21\n",
            "2006/2006-08-28\n",
            "2006/2006-09-04\n",
            "2006/2006-09-11\n",
            "2006/2006-09-18\n",
            "2006/2006-09-25\n",
            "2006/2006-10-02\n",
            "2006/2006-10-09\n",
            "2006/2006-10-16\n",
            "2006/2006-10-23\n",
            "2006/2006-10-30\n",
            "2006/2006-11-06\n",
            "2006/2006-11-13\n",
            "2006/2006-11-27\n",
            "2006/2006-12-11\n",
            "2006/2006-12-18\n",
            "2006/2006-12-25\n",
            "2007\n",
            "2007/2007-01-22\n",
            "2007/2007-02-05\n",
            "2007/2007-02-12\n",
            "2007/2007-02-19\n",
            "2007/2007-02-26\n",
            "2007/2007-03-12\n",
            "2007/2007-03-19\n",
            "2007/2007-03-26\n",
            "2007/2007-04-02\n",
            "2007/2007-04-09\n",
            "2007/2007-04-16\n",
            "2007/2007-04-23\n",
            "2007/2007-04-30\n",
            "2007/2007-05-07\n",
            "2007/2007-05-21\n",
            "2007/2007-05-28\n",
            "2007/2007-06-04\n",
            "2007/2007-06-11\n",
            "2007/2007-06-18\n",
            "2007/2007-06-25\n",
            "2007/2007-07-02\n",
            "2007/2007-07-09\n",
            "2007/2007-07-23\n",
            "2007/2007-07-30\n",
            "2007/2007-08-06\n",
            "2007/2007-08-27\n",
            "2007/2007-09-03\n",
            "2007/2007-09-10\n",
            "2007/2007-09-24\n",
            "2007/2007-10-01\n",
            "2007/2007-10-15\n",
            "2007/2007-10-22\n",
            "2007/2007-10-29\n",
            "2007/2007-11-05\n",
            "2007/2007-11-12\n",
            "2007/2007-11-19\n",
            "2007/2007-12-03\n",
            "2007/2007-12-10\n",
            "2007/2007-12-17\n",
            "2007/2007-12-24\n",
            "2007/2007-12-31\n",
            "2008\n",
            "2008/2008-01-14\n",
            "2008/2008-01-21\n",
            "2008/2008-01-28\n",
            "2008/2008-02-04\n",
            "2008/2008-02-11\n",
            "2008/2008-02-18\n",
            "2008/2008-02-25\n",
            "2008/2008-03-10\n",
            "2008/2008-03-17\n",
            "2008/2008-03-24\n",
            "2008/2008-03-31\n",
            "2008/2008-04-07\n",
            "2008/2008-04-21\n",
            "2008/2008-04-28\n",
            "2008/2008-05-05\n",
            "2008/2008-05-19\n",
            "2008/2008-05-26\n",
            "2008/2008-06-02\n",
            "2008/2008-06-09\n",
            "2008/2008-06-16\n",
            "2008/2008-06-30\n",
            "2008/2008-07-07\n",
            "2008/2008-07-14\n",
            "2008/2008-07-21\n",
            "2008/2008-07-28\n",
            "2008/2008-08-04\n",
            "2008/2008-08-11\n",
            "2008/2008-08-18\n",
            "2008/2008-08-25\n",
            "2008/2008-09-01\n",
            "2008/2008-09-08\n",
            "2008/2008-09-15\n",
            "2008/2008-09-22\n",
            "2008/2008-09-29\n",
            "2008/2008-10-27\n",
            "2008/2008-11-10\n",
            "2008/2008-11-17\n",
            "2008/2008-11-24\n",
            "2008/2008-12-08\n",
            "2008/2008-12-15\n",
            "2008/2008-12-22\n",
            "2009\n",
            "2009/2009-01-26\n",
            "2009/2009-02-02\n",
            "2009/2009-02-09\n",
            "2009/2009-02-23\n",
            "2009/2009-03-02\n",
            "2009/2009-03-09\n",
            "2009/2009-03-16\n",
            "2009/2009-03-30\n",
            "2009/2009-04-06\n",
            "2009/2009-04-13\n",
            "2009/2009-04-20\n",
            "2009/2009-04-27\n",
            "2009/2009-05-18\n",
            "2009/2009-05-25\n",
            "2009/2009-06-01\n",
            "2009/2009-06-15\n",
            "2009/2009-06-22\n",
            "2009/2009-06-29\n",
            "2009/2009-07-06\n",
            "2009/2009-07-13\n",
            "2009/2009-07-20\n",
            "2009/2009-07-27\n",
            "2009/2009-08-03\n",
            "2009/2009-08-10\n",
            "2009/2009-08-17\n",
            "2009/2009-08-24\n",
            "2009/2009-08-31\n",
            "2009/2009-09-07\n",
            "2009/2009-09-21\n",
            "2009/2009-09-28\n",
            "2009/2009-10-19\n",
            "2009/2009-11-02\n",
            "2009/2009-11-30\n",
            "2009/2009-12-07\n",
            "2009/2009-12-14\n",
            "2010\n",
            "2010/2010-01-25\n",
            "2010/2010-02-01\n",
            "2010/2010-02-15\n",
            "2010/2010-03-01\n",
            "2010/2010-03-08\n",
            "2010/2010-03-15\n",
            "2010/2010-03-22\n",
            "2010/2010-03-29\n",
            "2010/2010-04-05\n",
            "2010/2010-04-12\n",
            "2010/2010-04-19\n",
            "2010/2010-04-26\n",
            "2010/2010-05-03\n",
            "2010/2010-05-17\n",
            "2010/2010-05-24\n",
            "2010/2010-06-07\n",
            "2010/2010-06-14\n",
            "2010/2010-06-21\n",
            "2010/2010-06-28\n",
            "2010/2010-07-19\n",
            "2010/2010-08-23\n",
            "2010/2010-08-30\n",
            "2010/2010-09-13\n",
            "2010/2010-09-27\n",
            "2010/2010-10-11\n",
            "2010/2010-10-25\n",
            "2010/2010-11-15\n",
            "2010/2010-11-22\n",
            "2010/2010-11-29\n",
            "2010/2010-12-13\n",
            "2010/2010-12-20\n",
            "2011\n",
            "2011/2011-01-17\n",
            "2011/2011-01-24\n",
            "2011/2011-01-31\n",
            "2011/2011-02-14\n",
            "2011/2011-02-28\n",
            "2011/2011-03-14\n",
            "2011/2011-03-21\n",
            "2011/2011-03-28\n",
            "2011/2011-04-04\n",
            "2011/2011-04-18\n",
            "2011/2011-05-09\n",
            "2011/2011-05-23\n",
            "2011/2011-06-06\n",
            "2011/2011-06-13\n",
            "2011/2011-06-20\n",
            "2011/2011-06-27\n",
            "2011/2011-07-04\n",
            "2011/2011-07-11\n",
            "2011/2011-08-08\n",
            "2011/2011-08-22\n",
            "2011/2011-09-19\n",
            "2011/2011-10-24\n",
            "2011/2011-10-31\n",
            "2011/2011-11-14\n",
            "2011/2011-11-21\n",
            "2011/2011-11-28\n",
            "2011/2011-12-05\n",
            "2011/2011-12-12\n",
            "2012\n",
            "2012/2012-01-16\n",
            "2012/2012-01-23\n",
            "2012/2012-01-30\n",
            "2012/2012-02-06\n",
            "2012/2012-02-13\n",
            "2012/2012-02-20\n",
            "2012/2012-03-12\n",
            "2012/2012-03-19\n",
            "2012/2012-04-02\n",
            "2012/2012-04-09\n",
            "2012/2012-04-16\n",
            "2012/2012-04-23\n",
            "2012/2012-05-07\n",
            "2012/2012-05-14\n",
            "2012/2012-05-28\n",
            "2012/2012-06-04\n",
            "2012/2012-06-11\n",
            "2012/2012-06-18\n",
            "2012/2012-07-02\n",
            "2012/2012-07-09\n",
            "2012/2012-07-16\n",
            "2012/2012-07-23\n",
            "2012/2012-07-30\n",
            "2012/2012-08-06\n",
            "2012/2012-08-13\n",
            "2012/2012-08-20\n",
            "2012/2012-08-27\n",
            "2012/2012-09-03\n",
            "2012/2012-09-10\n",
            "2012/2012-09-24\n",
            "2012/2012-10-01\n",
            "2012/2012-10-08\n",
            "2012/2012-10-15\n",
            "2012/2012-10-22\n",
            "2012/2012-10-29\n",
            "2012/2012-11-05\n",
            "2012/2012-11-12\n",
            "2012/2012-11-19\n",
            "2012/2012-11-26\n",
            "2012/2012-12-03\n",
            "2012/2012-12-17\n",
            "2012/2012-12-24\n",
            "2012/2012-12-31\n",
            "2013\n",
            "2013/2013-01-14\n",
            "2013/2013-01-21\n",
            "2013/2013-01-28\n",
            "2013/2013-02-04\n",
            "2013/2013-02-11\n",
            "2013/2013-03-04\n",
            "2013/2013-03-11\n",
            "2013/2013-03-18\n",
            "2013/2013-04-01\n",
            "2013/2013-04-08\n",
            "2013/2013-04-22\n",
            "2013/2013-05-13\n",
            "2013/2013-05-20\n",
            "2013/2013-05-27\n",
            "2013/2013-06-03\n",
            "2013/2013-06-10\n",
            "2013/2013-06-17\n",
            "2013/2013-06-24\n",
            "2013/2013-07-01\n",
            "2013/2013-07-08\n",
            "2013/2013-07-22\n",
            "2013/2013-07-29\n",
            "2013/2013-08-05\n",
            "2013/2013-08-12\n",
            "2013/2013-08-19\n",
            "2013/2013-08-26\n",
            "2013/2013-09-09\n",
            "2013/2013-09-16\n",
            "2013/2013-09-23\n",
            "2013/2013-09-30\n",
            "2013/2013-10-07\n",
            "2013/2013-10-14\n",
            "2013/2013-10-21\n",
            "2013/2013-10-28\n",
            "2013/2013-11-04\n",
            "2013/2013-11-11\n",
            "2013/2013-11-18\n",
            "2013/2013-11-25\n",
            "2013/2013-12-02\n",
            "2013/2013-12-09\n",
            "2013/2013-12-16\n",
            "2013/2013-12-23\n",
            "2013/2013-12-30\n",
            "2014\n",
            "2014/2014-01-20\n",
            "2014/2014-02-17\n",
            "2014/2014-03-10\n",
            "2014/2014-03-17\n",
            "2014/2014-04-07\n",
            "2014/2014-04-21\n",
            "2014/2014-04-28\n",
            "2014/2014-05-19\n",
            "2014/2014-06-02\n",
            "2014/2014-06-09\n",
            "2014/2014-06-23\n",
            "2014/2014-11-03\n",
            "2014/2014-11-24\n",
            "2014/2014-12-15\n",
            "2014/2014-12-29\n",
            "2014/2014-01-27\n",
            "2014/2014-02-03\n",
            "2014/2014-02-24\n",
            "2014/2014-03-03\n",
            "2014/2014-03-24\n",
            "2014/2014-05-05\n",
            "2014/2014-05-26\n",
            "2014/2014-06-30\n",
            "2014/2014-07-07\n",
            "2014/2014-07-21\n",
            "2014/2014-07-28\n",
            "2014/2014-08-04\n",
            "2014/2014-08-18\n",
            "2014/2014-08-25\n",
            "2014/2014-09-01\n",
            "2014/2014-09-08\n",
            "2014/2014-09-22\n",
            "2014/2014-09-29\n",
            "2014/2014-10-06\n",
            "2014/2014-10-13\n",
            "2014/2014-10-20\n",
            "2014/2014-10-27\n",
            "2014/2014-12-22\n",
            "2015\n",
            "2015/2015-01-19\n",
            "2015/2015-01-26\n",
            "2015/2015-02-02\n",
            "2015/2015-02-09\n",
            "2015/2015-02-16\n",
            "2015/2015-03-09\n",
            "2015/2015-03-16\n",
            "2015/2015-03-23\n",
            "2015/2015-03-30\n",
            "2015/2015-04-06\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hjlWRRmKqJmI"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sYqOxGMr9tLe"
      },
      "source": [
        "# What is on the cover page?\n",
        "The most Frequent Terms"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tJJE2-WdqJrK",
        "outputId": "05ebcfbf-99a1-45c2-bc0a-672cfade0f67"
      },
      "source": [
        "try:    from unidecode import unidecode\n",
        "except:\n",
        "    !pip install unidecode\n",
        "    from unidecode import unidecode\n",
        "\n",
        "\n",
        "def read_text_file(filename):\n",
        "    with open(filename) as f:    \n",
        "        content = ' '.join([unidecode(i.strip())   for i in f.readlines()   if i.strip()]).replace('- ', '')\n",
        "    return content\n",
        "\n",
        "def get_magazine_text(path):\n",
        "    pages  =  sorted([os.path.join(path, file)   for file in os.listdir(path)   if file.endswith('.txt')], key=lambda x: int(x.split('/')[-1][:-4]))\n",
        "    texts  =  [read_text_file(page)  for page in pages]\n",
        "    return texts\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting unidecode\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9e/25/723487ca2a52ebcee88a34d7d1f5a4b80b793f179ee0f62d5371938dfa01/Unidecode-1.2.0-py2.py3-none-any.whl (241kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 3.9MB/s \n",
            "\u001b[?25hInstalling collected packages: unidecode\n",
            "Successfully installed unidecode-1.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dCzBALwYyObI"
      },
      "source": [
        "release_dates  =  sum([[os.path.join(str(year), dir_)  for dir_ in os.listdir(str(year))  if '.' not in dir_]  for year in range(2000, 2016)], [])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EEDFs-oW0E7H"
      },
      "source": [
        "texts = []\n",
        "for  release_date  in release_dates:\n",
        "    # print('\\n\\n\\n', release_date)\n",
        "    text = read_text_file(release_date+'/1.txt')\n",
        "    texts.append(text)\n",
        "    # print(text)\n",
        "\n",
        "    # doc = nlp(text)\n",
        "    # for entity in doc.ents:\n",
        "    #     print(entity.label_, ' | ', entity.text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EpHDI5QO7gEq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3678b235-527b-4af1-d98b-b016742b4e52"
      },
      "source": [
        "from collections import Counter\n",
        "l = ' '.join(texts).lower().split()\n",
        "l = [i  for i in l  if len(i)>=3 and i not in stopword_set]\n",
        "c = Counter(l)\n",
        "c.most_common()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('new', 75),\n",
              " ('joe', 57),\n",
              " ('eee', 56),\n",
              " ('time', 55),\n",
              " ('special', 49),\n",
              " ('inside', 39),\n",
              " ('michael', 39),\n",
              " ('klein', 38),\n",
              " ('america', 33),\n",
              " ('world', 28),\n",
              " ('plus:', 28),\n",
              " ('obama', 28),\n",
              " ('report', 27),\n",
              " ('david', 27),\n",
              " ('see', 24),\n",
              " ('health', 24),\n",
              " ('page', 24),\n",
              " ('one', 23),\n",
              " ('eae', 22),\n",
              " ('u.s.', 22),\n",
              " ('make', 21),\n",
              " ('war', 21),\n",
              " ('issue', 20),\n",
              " ('von', 19),\n",
              " ('drehle', 18),\n",
              " ('science', 17),\n",
              " ('behind', 17),\n",
              " ('look', 17),\n",
              " ('best', 16),\n",
              " (\"america's\", 16),\n",
              " ('barack', 16),\n",
              " (\"what's\", 15),\n",
              " ('bush', 15),\n",
              " ('way', 15),\n",
              " ('real', 15),\n",
              " ('exclusive', 15),\n",
              " ('president', 15),\n",
              " ('klein:', 15),\n",
              " ('john', 14),\n",
              " ('(c)', 14),\n",
              " ('change', 14),\n",
              " ('want', 14),\n",
              " ('care', 14),\n",
              " ('like', 13),\n",
              " ('could', 13),\n",
              " ('really', 13),\n",
              " ('american', 13),\n",
              " (\"he's\", 13),\n",
              " ('save', 13),\n",
              " ('fareed', 13),\n",
              " ('future', 12),\n",
              " ('still', 12),\n",
              " ('aaa', 12),\n",
              " ('life', 12),\n",
              " ('need', 12),\n",
              " ('guide', 12),\n",
              " ('election', 12),\n",
              " ('economy', 12),\n",
              " (\"obama's\", 12),\n",
              " ('battle', 11),\n",
              " ('case', 11),\n",
              " ('politics', 11),\n",
              " ('get', 11),\n",
              " ('made', 11),\n",
              " ('ate', 11),\n",
              " ('big', 11),\n",
              " ('latest', 11),\n",
              " ('history', 11),\n",
              " ('may', 11),\n",
              " (\"here's\", 11),\n",
              " ('man', 11),\n",
              " ('men', 11),\n",
              " ('good', 11),\n",
              " ('year', 11),\n",
              " ('march', 11),\n",
              " ('plan', 11),\n",
              " ('office', 11),\n",
              " ('back', 11),\n",
              " ('next', 11),\n",
              " ('debate', 10),\n",
              " ('kids', 10),\n",
              " ('five', 10),\n",
              " ('deg', 10),\n",
              " ('bill', 10),\n",
              " ('ele', 10),\n",
              " (\"bush's\", 10),\n",
              " ('vs.', 10),\n",
              " ('pee', 10),\n",
              " ('people', 10),\n",
              " ('campaign', 10),\n",
              " ('better', 10),\n",
              " ('making', 10),\n",
              " ('plus', 10),\n",
              " ('food', 10),\n",
              " ('rana', 10),\n",
              " ('foroohar', 10),\n",
              " ('mee', 9),\n",
              " ('americans', 9),\n",
              " ('aes', 9),\n",
              " ('would', 9),\n",
              " ('movie', 9),\n",
              " ('tee', 9),\n",
              " ('end', 9),\n",
              " ('team', 9),\n",
              " ('tae', 9),\n",
              " ('got', 9),\n",
              " ('means', 9),\n",
              " (\"who's\", 9),\n",
              " ('crisis', 9),\n",
              " ('jeffrey', 9),\n",
              " ('karen', 9),\n",
              " ('global', 9),\n",
              " ('top', 9),\n",
              " ('mccain', 9),\n",
              " ('joel', 9),\n",
              " ('zakaria', 9),\n",
              " ('tet', 8),\n",
              " ('ere', 8),\n",
              " ('tale', 8),\n",
              " ('sex', 8),\n",
              " ('gates', 8),\n",
              " ('school', 8),\n",
              " ('--_', 8),\n",
              " ('service', 8),\n",
              " ('china', 8),\n",
              " ('faith', 8),\n",
              " ('says', 8),\n",
              " ('age', 8),\n",
              " ('power', 8),\n",
              " ('story', 8),\n",
              " ('many', 8),\n",
              " ('nancy', 8),\n",
              " ('cee', 8),\n",
              " ('first', 8),\n",
              " ('republican', 8),\n",
              " ('middle', 8),\n",
              " ('eat', 8),\n",
              " ('hillary', 8),\n",
              " ('use', 8),\n",
              " ('report:', 8),\n",
              " ('race', 8),\n",
              " ('www.time.com', 8),\n",
              " ('clinton', 8),\n",
              " ('tumulty', 8),\n",
              " ('wall', 8),\n",
              " ('scherer', 8),\n",
              " ('january', 7),\n",
              " ('james', 7),\n",
              " ('books', 7),\n",
              " ('paul', 7),\n",
              " ('research', 7),\n",
              " ('ete', 7),\n",
              " ('last', 7),\n",
              " ('home', 7),\n",
              " ('george', 7),\n",
              " ('system', 7),\n",
              " ('think', 7),\n",
              " ('mind', 7),\n",
              " (\"we're\", 7),\n",
              " ('(tm)', 7),\n",
              " ('ane', 7),\n",
              " ('star', 7),\n",
              " ('tells', 7),\n",
              " ('nation', 7),\n",
              " ('lee', 7),\n",
              " ('women', 7),\n",
              " ('inside:', 7),\n",
              " ('secret', 7),\n",
              " ('eer', 7),\n",
              " (\"world's\", 7),\n",
              " ('high', 7),\n",
              " ('pakistan', 7),\n",
              " ('nee', 7),\n",
              " ('movies', 7),\n",
              " ('west', 7),\n",
              " ('going', 7),\n",
              " ('wrong', 7),\n",
              " ('kluger', 7),\n",
              " ('game', 7),\n",
              " ('right', 7),\n",
              " ('revolution', 7),\n",
              " ('win', 7),\n",
              " ('takes', 7),\n",
              " ('know', 7),\n",
              " ('2008', 7),\n",
              " ('national', 7),\n",
              " ('fix', 7),\n",
              " ('fight', 7),\n",
              " ('work', 7),\n",
              " ('romney', 7),\n",
              " ('time.com', 7),\n",
              " ('steve', 6),\n",
              " ('cae', 6),\n",
              " ('washington', 6),\n",
              " ('shows', 6),\n",
              " ('ever', 6),\n",
              " ('million', 6),\n",
              " ('makes', 6),\n",
              " ('parents', 6),\n",
              " ('(r)', 6),\n",
              " ('sae', 6),\n",
              " (\"can't\", 6),\n",
              " ('years', 6),\n",
              " ('scientists', 6),\n",
              " ('say', 6),\n",
              " (\"they're\", 6),\n",
              " ('bin', 6),\n",
              " ('gibbs', 6),\n",
              " ('jobs', 6),\n",
              " ('eel', 6),\n",
              " ('biggest', 6),\n",
              " ('mission', 6),\n",
              " ('aie', 6),\n",
              " ('9/11', 6),\n",
              " ('leader', 6),\n",
              " ('iraq', 6),\n",
              " ('err', 6),\n",
              " ('teach', 6),\n",
              " ('brain', 6),\n",
              " ('god', 6),\n",
              " ('democrats', 6),\n",
              " ('much', 6),\n",
              " ('claudia', 6),\n",
              " ('lessons', 6),\n",
              " ('point', 6),\n",
              " ('tea', 6),\n",
              " ('talk', 6),\n",
              " ('culture', 6),\n",
              " ('cloud', 6),\n",
              " ('help', 6),\n",
              " ('changed', 6),\n",
              " ('annual', 6),\n",
              " ('iran', 6),\n",
              " ('forget', 6),\n",
              " ('richard', 6),\n",
              " ('mark', 6),\n",
              " ('grunwald', 6),\n",
              " ('schools', 6),\n",
              " ('bryan', 6),\n",
              " ('walsh', 6),\n",
              " ('alice', 6),\n",
              " ('2010', 6),\n",
              " ('2012', 6),\n",
              " ('crowley', 6),\n",
              " ('$5.99can', 6),\n",
              " ('century', 5),\n",
              " ('money', 5),\n",
              " ('een', 5),\n",
              " ('ees', 5),\n",
              " ('king', 5),\n",
              " ('space', 5),\n",
              " ('watch', 5),\n",
              " ('final', 5),\n",
              " ('works', 5),\n",
              " ('bes', 5),\n",
              " ('hes', 5),\n",
              " ('come', 5),\n",
              " ('millions', 5),\n",
              " ('private', 5),\n",
              " ('ata', 5),\n",
              " ('als', 5),\n",
              " ('spy', 5),\n",
              " ('two', 5),\n",
              " ('yes,', 5),\n",
              " ('education', 5),\n",
              " ('court', 5),\n",
              " ('came', 5),\n",
              " ('eed', 5),\n",
              " ('ttt', 5),\n",
              " ('lives', 5),\n",
              " ('sea', 5),\n",
              " ('ine', 5),\n",
              " ('let', 5),\n",
              " ('ree', 5),\n",
              " ('billion', 5),\n",
              " ('res', 5),\n",
              " ('journey', 5),\n",
              " ('rea', 5),\n",
              " ('gee', 5),\n",
              " ('ran', 5),\n",
              " ('yee', 5),\n",
              " ('three', 5),\n",
              " ('coe', 5),\n",
              " ('living', 5),\n",
              " ('bad', 5),\n",
              " ('nae', 5),\n",
              " ('young', 5),\n",
              " ('getting', 5),\n",
              " ('seu', 5),\n",
              " ('olympic', 5),\n",
              " ('place', 5),\n",
              " ('tech', 5),\n",
              " ('live', 5),\n",
              " ('dangerous', 5),\n",
              " ('wallis', 5),\n",
              " ('secrets', 5),\n",
              " ('eur', 5),\n",
              " ('investigation', 5),\n",
              " ('war?', 5),\n",
              " ('cost', 5),\n",
              " ('party', 5),\n",
              " ('summer', 5),\n",
              " ('white', 5),\n",
              " ('duffy', 5),\n",
              " ('never', 5),\n",
              " ('cancer', 5),\n",
              " ('changing', 5),\n",
              " ('street', 5),\n",
              " ('problem', 5),\n",
              " ('scenes', 5),\n",
              " ('stein', 5),\n",
              " (\"china's\", 5),\n",
              " ('park', 5),\n",
              " (\"there's\", 5),\n",
              " ('movies:', 5),\n",
              " ('saporito', 5),\n",
              " ('dr.', 5),\n",
              " ('move', 5),\n",
              " ('mitt', 5),\n",
              " ('$4.99us', 5),\n",
              " ('hee', 4),\n",
              " ('lost', 4),\n",
              " ('ing', 4),\n",
              " ('seen', 4),\n",
              " ('pope', 4),\n",
              " ('wie', 4),\n",
              " ('peer', 4),\n",
              " ('dad', 4),\n",
              " ('later', 4),\n",
              " ('ers', 4),\n",
              " ('ucr', 4),\n",
              " ('eens', 4),\n",
              " ('hit', 4),\n",
              " ('letters', 4),\n",
              " ('bae', 4),\n",
              " ('ene', 4),\n",
              " ('tey', 4),\n",
              " ('greatest', 4),\n",
              " ('quest', 4),\n",
              " ('myths', 4),\n",
              " ('marriage', 4),\n",
              " ('leaders', 4),\n",
              " ('aoe', 4),\n",
              " ('finally', 4),\n",
              " ('aay', 4),\n",
              " ('actually', 4),\n",
              " ('ways', 4),\n",
              " ('find', 4),\n",
              " ('wee', 4),\n",
              " ('failure', 4),\n",
              " ('utr', 4),\n",
              " ('aye', 4),\n",
              " ('teen', 4),\n",
              " ('beyond', 4),\n",
              " ('wat', 4),\n",
              " ('earth', 4),\n",
              " ('rae', 4),\n",
              " ('feel', 4),\n",
              " ('karl', 4),\n",
              " ('pur', 4),\n",
              " ('sas', 4),\n",
              " ('sage', 4),\n",
              " ('ess', 4),\n",
              " ('nuclear', 4),\n",
              " ('fbi', 4),\n",
              " ('ent', 4),\n",
              " ('set', 4),\n",
              " ('civil', 4),\n",
              " ('taliban', 4),\n",
              " ('treatments', 4),\n",
              " ('ona', 4),\n",
              " ('ery', 4),\n",
              " ('soe', 4),\n",
              " ('bible', 4),\n",
              " ('roe', 4),\n",
              " ('stop', 4),\n",
              " ('les', 4),\n",
              " ('uss', 4),\n",
              " ('things', 4),\n",
              " ('ideas', 4),\n",
              " ('master', 4),\n",
              " ('turning', 4),\n",
              " ('take', 4),\n",
              " ('him?', 4),\n",
              " ('disease', 4),\n",
              " ('donald', 4),\n",
              " ('barlett', 4),\n",
              " ('steele', 4),\n",
              " ('sean', 4),\n",
              " ('beat', 4),\n",
              " ('already', 4),\n",
              " ('east', 4),\n",
              " ('family', 4),\n",
              " ('video', 4),\n",
              " ('book', 4),\n",
              " ('att', 4),\n",
              " ('run', 4),\n",
              " ('girls', 4),\n",
              " ('generation', 4),\n",
              " ('com', 4),\n",
              " (\"they'll\", 4),\n",
              " ('interview', 4),\n",
              " ('done', 4),\n",
              " ('sse', 4),\n",
              " ('athletes', 4),\n",
              " ('struggle', 4),\n",
              " ('higher', 4),\n",
              " ('november', 4),\n",
              " ('hot', 4),\n",
              " ('longer', 4),\n",
              " ('lev', 4),\n",
              " ('grossman', 4),\n",
              " ('boss', 4),\n",
              " ('house', 4),\n",
              " ('deal', 4),\n",
              " ('ely', 4),\n",
              " ('preview', 4),\n",
              " ('went', 4),\n",
              " ('put', 4),\n",
              " ('worst', 4),\n",
              " ('scandal', 4),\n",
              " ('-and', 4),\n",
              " ('ial', 4),\n",
              " ('long', 4),\n",
              " ('oscar', 4),\n",
              " ('pot', 4),\n",
              " ('killing', 4),\n",
              " ('meltdown', 4),\n",
              " ('sullivan', 4),\n",
              " ('humans', 4),\n",
              " ('days', 4),\n",
              " ('day', 4),\n",
              " ('became', 4),\n",
              " ('tim', 4),\n",
              " ('harry', 4),\n",
              " ('keep', 4),\n",
              " ('religion', 4),\n",
              " ('petraeus', 4),\n",
              " ('isa', 4),\n",
              " ('machine', 4),\n",
              " ('broken', 4),\n",
              " ('photos', 4),\n",
              " ('myth', 4),\n",
              " ('texas', 4),\n",
              " ('justin', 4),\n",
              " ('mad', 4),\n",
              " ('meaning', 4),\n",
              " ('financial', 4),\n",
              " ('mess', 4),\n",
              " ('elizabeth', 4),\n",
              " ('needs', 4),\n",
              " ('tragedy', 4),\n",
              " ('fighting', 4),\n",
              " ('ahead', 4),\n",
              " ('rise', 4),\n",
              " ('states', 4),\n",
              " ('barton', 4),\n",
              " ('jon', 4),\n",
              " ('michele', 4),\n",
              " ('2013', 4),\n",
              " ('2014', 4),\n",
              " ('welcome', 3),\n",
              " ('toa', 3),\n",
              " ('per', 3),\n",
              " ('city', 3),\n",
              " ('ert', 3),\n",
              " ('movies,', 3),\n",
              " ('ith', 3),\n",
              " ('tees', 3),\n",
              " ('ste', 3),\n",
              " ('ptt', 3),\n",
              " ('ett', 3),\n",
              " ('suc', 3),\n",
              " ('ooo', 3),\n",
              " ('tom', 3),\n",
              " ('rudy', 3),\n",
              " ('-ee', 3),\n",
              " ('puts', 3),\n",
              " ('dee', 3),\n",
              " ('ese', 3),\n",
              " ('drugs', 3),\n",
              " ('le)', 3),\n",
              " ('become', 3),\n",
              " ('gore', 3),\n",
              " ('tat', 3),\n",
              " ('wars', 3),\n",
              " ('hospital', 3),\n",
              " ('stay', 3),\n",
              " ('19,', 3),\n",
              " ('lot', 3),\n",
              " ('ces', 3),\n",
              " ('eas', 3),\n",
              " ('try', 3),\n",
              " ('ae)', 3),\n",
              " ('es)', 3),\n",
              " ('eter', 3),\n",
              " ('oto', 3),\n",
              " ('ten', 3),\n",
              " ('amazing', 3),\n",
              " ('happen', 3),\n",
              " ('incredible', 3),\n",
              " ('shrinking', 3),\n",
              " ('oes', 3),\n",
              " ('test', 3),\n",
              " ('effect', 3),\n",
              " ('others', 3),\n",
              " ('afraid', 3),\n",
              " ('warming', 3),\n",
              " ('lone', 3),\n",
              " ('wort', 3),\n",
              " ('blues', 3),\n",
              " ('smc', 3),\n",
              " ('revolutionary', 3),\n",
              " ('eam', 3),\n",
              " ('wey', 3),\n",
              " ('universe', 3),\n",
              " ('mystery', 3),\n",
              " ('--s', 3),\n",
              " ('pay', 3),\n",
              " ('control', 3),\n",
              " ('human', 3),\n",
              " ('growing', 3),\n",
              " ('pushing', 3),\n",
              " ('uma', 3),\n",
              " ('suicide', 3),\n",
              " ('laden', 3),\n",
              " ('without', 3),\n",
              " ('front', 3),\n",
              " ('week', 3),\n",
              " ('strength', 3),\n",
              " ('smartest', 3),\n",
              " ('figure', 3),\n",
              " ('daniel', 3),\n",
              " ('impossible', 3),\n",
              " ('ae.', 3),\n",
              " ('ure', 3),\n",
              " ('wae', 3),\n",
              " ('play', 3),\n",
              " ('learn', 3),\n",
              " (\"hollywood's\", 3),\n",
              " ('taking', 3),\n",
              " ('woman', 3),\n",
              " ('do?', 3),\n",
              " ('ofthe', 3),\n",
              " ('giant', 3),\n",
              " ('hunt', 3),\n",
              " ('senate', 3),\n",
              " ('jesus', 3),\n",
              " ('heats', 3),\n",
              " ('healthy', 3),\n",
              " ('coming', 3),\n",
              " ('aia', 3),\n",
              " ('ae]', 3),\n",
              " ('ait', 3),\n",
              " ('safe', 3),\n",
              " ('2003', 3),\n",
              " ('bono', 3),\n",
              " ('wit', 3),\n",
              " ('fee', 3),\n",
              " ('told', 3),\n",
              " (\"where's\", 3),\n",
              " ('costs', 3),\n",
              " ('excerpt', 3),\n",
              " ('public', 3),\n",
              " ('children', 3),\n",
              " ('eke', 3),\n",
              " ('state', 3),\n",
              " ('jan.', 3),\n",
              " ('28,', 3),\n",
              " ('since', 3),\n",
              " ('film', 3),\n",
              " ('sss', 3),\n",
              " ('ses', 3),\n",
              " ('sat', 3),\n",
              " ('tes', 3),\n",
              " ('boy', 3),\n",
              " ('925', 3),\n",
              " ('view', 3),\n",
              " ('blame', 3),\n",
              " ('exit', 3),\n",
              " ('strategy', 3),\n",
              " ('terrorism', 3),\n",
              " ('cheney', 3),\n",
              " ('aan', 3),\n",
              " ('anew', 3),\n",
              " ('influential', 3),\n",
              " ('afghanistan', 3),\n",
              " ('work?', 3),\n",
              " ('matters', 3),\n",
              " ('acu', 3),\n",
              " ('net', 3),\n",
              " ('tas', 3),\n",
              " ('town', 3),\n",
              " ('republicans', 3),\n",
              " ('crt', 3),\n",
              " ('goes', 3),\n",
              " ('united', 3),\n",
              " ('games', 3),\n",
              " ('ita', 3),\n",
              " ('10,', 3),\n",
              " ('hat', 3),\n",
              " ('dave', 3),\n",
              " ('job', 3),\n",
              " ('true', 3),\n",
              " ('room', 3),\n",
              " ('finding', 3),\n",
              " ('balance', 3),\n",
              " ('left', 3),\n",
              " ('serious', 3),\n",
              " ('intimate', 3),\n",
              " ('bring', 3),\n",
              " ('speaks', 3),\n",
              " ('everyone', 3),\n",
              " ('matt', 3),\n",
              " ('grand', 3),\n",
              " ('pie', 3),\n",
              " ('ies', 3),\n",
              " ('fall', 3),\n",
              " ('issue:', 3),\n",
              " ('mer', 3),\n",
              " ('aed', 3),\n",
              " ('fat', 3),\n",
              " ('apple', 3),\n",
              " ('ceo', 3),\n",
              " ('2006', 3),\n",
              " ('congress', 3),\n",
              " (\"'on\", 3),\n",
              " ('hea', 3),\n",
              " ('web', 3),\n",
              " ('wes', 3),\n",
              " ('found', 3),\n",
              " ('surprising', 3),\n",
              " ('google', 3),\n",
              " ('energy', 3),\n",
              " ('ath', 3),\n",
              " ('climate', 3),\n",
              " ('preview:', 3),\n",
              " ('tate', 3),\n",
              " ('einstein', 3),\n",
              " ('vie', 3),\n",
              " ('north', 3),\n",
              " ('presidential', 3),\n",
              " ('looks', 3),\n",
              " ('learned', 3),\n",
              " ('jack', 3),\n",
              " ('--,', 3),\n",
              " ('eve', 3),\n",
              " ('van', 3),\n",
              " ('biema', 3),\n",
              " ('jeff', 3),\n",
              " ('world.', 3),\n",
              " ('candidates', 3),\n",
              " ('director', 3),\n",
              " ('hear', 3),\n",
              " ('vice', 3),\n",
              " ('doctor', 3),\n",
              " ('legacy', 3),\n",
              " ('poniewozik', 3),\n",
              " ('success', 3),\n",
              " ('100', 3),\n",
              " ('potter', 3),\n",
              " ('ame', 3),\n",
              " ('reveal', 3),\n",
              " ('edwards', 3),\n",
              " ('grow', 3),\n",
              " ('nice', 3),\n",
              " ('failing', 3),\n",
              " ('gray', 3),\n",
              " ('action', 3),\n",
              " ('stengel', 3),\n",
              " ('questions', 3),\n",
              " ('health-care', 3),\n",
              " ('around', 3),\n",
              " ('phones', 3),\n",
              " ('building', 3),\n",
              " ('environment', 3),\n",
              " ('stem', 3),\n",
              " (\"gop's\", 3),\n",
              " ('bank', 3),\n",
              " ('ote', 3),\n",
              " ('survive', 3),\n",
              " ('jay', 3),\n",
              " ('president?', 3),\n",
              " ('ela', 3),\n",
              " ('sia', 3),\n",
              " ('late', 3),\n",
              " ('common', 3),\n",
              " ('prices', 3),\n",
              " ('college', 3),\n",
              " ('disaster', 3),\n",
              " ('michelle', 3),\n",
              " ('truth', 3),\n",
              " ('ripley', 3),\n",
              " ('hal', 3),\n",
              " ('great', 3),\n",
              " ('sex?', 3),\n",
              " ('richer', 3),\n",
              " ('car', 3),\n",
              " ('strong', 3),\n",
              " ('inventions', 3),\n",
              " ('hae', 3),\n",
              " ('saving', 3),\n",
              " ('gone', 3),\n",
              " ('wild', 3),\n",
              " ('mom', 3),\n",
              " ('commemorative', 3),\n",
              " ('tales', 3),\n",
              " ('toxic', 3),\n",
              " ('choices', 3),\n",
              " ('aid', 3),\n",
              " ('flu', 3),\n",
              " ('old', 3),\n",
              " ('reform', 3),\n",
              " ('massimo', 3),\n",
              " ('calabresi', 3),\n",
              " ('stein:', 3),\n",
              " ('scare', 3),\n",
              " ('night', 3),\n",
              " ('decision', 3),\n",
              " ('newt', 3),\n",
              " ('bobby', 3),\n",
              " ('ghosh', 3),\n",
              " ('art', 3),\n",
              " ('rand', 3),\n",
              " ('double', 3),\n",
              " ('gop', 3),\n",
              " ('gellman', 3),\n",
              " ('zakaria:', 3),\n",
              " ('julian', 3),\n",
              " ('ory', 3),\n",
              " ('tel', 3),\n",
              " ('lincoln', 3),\n",
              " ('risks', 3),\n",
              " ('season', 3),\n",
              " ('mal', 3),\n",
              " ('core', 3),\n",
              " ('arte', 3),\n",
              " ('new,', 3),\n",
              " ('sees', 3),\n",
              " ('mobile', 3),\n",
              " ('knows', 3),\n",
              " ('poll', 3),\n",
              " ('fire', 3),\n",
              " ('10090', 3),\n",
              " ('hus', 3),\n",
              " ('/the', 3),\n",
              " ('\"\"92567\"10090\\'', 3),\n",
              " (\"lran's\", 3),\n",
              " ('yellen', 3),\n",
              " ('01100011', 3),\n",
              " ('a\".', 2),\n",
              " ('york', 2),\n",
              " ('laws', 2),\n",
              " ('jerry', 2),\n",
              " ('cur', 2),\n",
              " ('925675\"0', 2),\n",
              " ('val', 2),\n",
              " ('ence', 2),\n",
              " ('hang', 2),\n",
              " ('hearts', 2),\n",
              " ('husband', 2),\n",
              " ('ucu', 2),\n",
              " ('stat', 2),\n",
              " ('function', 2),\n",
              " ('form', 2),\n",
              " ('moe', 2),\n",
              " ('12>', 2),\n",
              " ('por', 2),\n",
              " ('aids', 2),\n",
              " ('hollywood', 2),\n",
              " ('ral', 2),\n",
              " ('lor', 2),\n",
              " ('ays', 2),\n",
              " ('erie', 2),\n",
              " ('etm', 2),\n",
              " ('political', 2),\n",
              " ('soon', 2),\n",
              " ('also', 2),\n",
              " ('ale', 2),\n",
              " ('april', 2),\n",
              " ('hours', 2),\n",
              " ('pes', 2),\n",
              " ('cies', 2),\n",
              " ('years.', 2),\n",
              " ('fresh', 2),\n",
              " ('microsoft', 2),\n",
              " ('hacker', 2),\n",
              " ('careers', 2),\n",
              " ('crises', 2),\n",
              " ('\"ie', 2),\n",
              " ('dow', 2),\n",
              " ('ret', 2),\n",
              " ('caleb', 2),\n",
              " ('rus', 2),\n",
              " ('tit', 2),\n",
              " ('ie)', 2),\n",
              " ('smart', 2),\n",
              " ('cars,', 2),\n",
              " ('robots', 2),\n",
              " ('ready?', 2),\n",
              " ('novel', 2),\n",
              " ('emai', 2),\n",
              " ('brother', 2),\n",
              " ('francis', 2),\n",
              " ('collins', 2),\n",
              " ('feat', 2),\n",
              " ('esn', 2),\n",
              " ('es:', 2),\n",
              " ('...', 2),\n",
              " ('atin', 2),\n",
              " ('mere', 2),\n",
              " ('jeb', 2),\n",
              " ('golfer', 2),\n",
              " (\"gore's\", 2),\n",
              " ('historic', 2),\n",
              " ('leap', 2),\n",
              " ('ars', 2),\n",
              " ('34>', 2),\n",
              " ('tek', 2),\n",
              " ('eles', 2),\n",
              " ('happy?', 2),\n",
              " ('u.n.', 2),\n",
              " ('jones', 2),\n",
              " ('wants', 2),\n",
              " ('it?', 2),\n",
              " ('spend', 2),\n",
              " ('alone', 2),\n",
              " ('unhappy', 2),\n",
              " ('upended', 2),\n",
              " ('tie', 2),\n",
              " ('hie', 2),\n",
              " ('reacts', 2),\n",
              " ('something', 2),\n",
              " ('coping', 2),\n",
              " ('naa', 2),\n",
              " ('cte', 2),\n",
              " ('yor', 2),\n",
              " ('saa', 2),\n",
              " ('els', 2),\n",
              " ('cert', 2),\n",
              " ('rrr', 2),\n",
              " ('pans', 2),\n",
              " ('eet', 2),\n",
              " ('lal', 2),\n",
              " ('undo', 2),\n",
              " ('damage', 2),\n",
              " ('causes', 2),\n",
              " ('read', 2),\n",
              " ('couples', 2),\n",
              " ('perils', 2),\n",
              " ('aad', 2),\n",
              " ('as)', 2),\n",
              " ('colleges', 2),\n",
              " ('famous', 2),\n",
              " ('(tm)~', 2),\n",
              " ('california', 2),\n",
              " ('killer', 2),\n",
              " ('silence', 2),\n",
              " ('jae', 2),\n",
              " ('thi', 2),\n",
              " ('early', 2),\n",
              " ('hundreds', 2),\n",
              " ('face', 2),\n",
              " ('up.', 2),\n",
              " ('feeling', 2),\n",
              " ('sige', 2),\n",
              " ('st.', 2),\n",
              " ('(and', 2),\n",
              " ('bob', 2),\n",
              " ('mur', 2),\n",
              " ('ila', 2),\n",
              " ('rue', 2),\n",
              " ('prevent', 2),\n",
              " ('hed', 2),\n",
              " ('breakthrough', 2),\n",
              " (\"we've\", 2),\n",
              " ('cue', 2),\n",
              " ('muu', 2),\n",
              " ('exclusive:', 2),\n",
              " ('blind', 2),\n",
              " ('protect', 2),\n",
              " ('cra', 2),\n",
              " ('aol', 2),\n",
              " ('keyword:', 2),\n",
              " ('rem', 2),\n",
              " ('discovery', 2),\n",
              " ('ares', 2),\n",
              " ('tmu', 2),\n",
              " ('part', 2),\n",
              " ('doctors', 2),\n",
              " ('biologist', 2),\n",
              " ('women,', 2),\n",
              " ('led', 2),\n",
              " ('center', 2),\n",
              " ('aer', 2),\n",
              " ('holding', 2),\n",
              " ('asl', 2),\n",
              " ('rut', 2),\n",
              " ('nas', 2),\n",
              " ('elt', 2),\n",
              " ('muslim', 2),\n",
              " ('world?', 2),\n",
              " ('anthrax', 2),\n",
              " ('fom', 2),\n",
              " ('ske', 2),\n",
              " ('heat', 2),\n",
              " ('manhunt', 2),\n",
              " ('pet', 2),\n",
              " ('invention', 2),\n",
              " ('|the', 2),\n",
              " ('tower', 2),\n",
              " ('lay', 2),\n",
              " ('mid', 2),\n",
              " ('thinks', 2),\n",
              " ('all-in-one', 2),\n",
              " ('music,', 2),\n",
              " ('pictures', 2),\n",
              " ('bst', 2),\n",
              " ('eos', 2),\n",
              " ('$3.95us', 2),\n",
              " ('25679', 2),\n",
              " ('cele', 2),\n",
              " ('death', 2),\n",
              " ('insidethe', 2),\n",
              " ('terrorist', 2),\n",
              " ('pees', 2),\n",
              " ('sel', 2),\n",
              " ('rat', 2),\n",
              " ('babies', 2),\n",
              " ('both?', 2),\n",
              " ('sey', 2),\n",
              " ('ues', 2),\n",
              " ('sur', 2),\n",
              " ('ose', 2),\n",
              " ('ewe', 2),\n",
              " ('ome', 2),\n",
              " ('tev', 2),\n",
              " ('oro', 2),\n",
              " ('-oe', 2),\n",
              " ('tse', 2),\n",
              " ('rowley', 2),\n",
              " ('details', 2),\n",
              " ('ler', 2),\n",
              " ('ons', 2),\n",
              " ('mum', 2),\n",
              " ('reading', 2),\n",
              " ('reasons', 2),\n",
              " ('toe', 2),\n",
              " ('enough', 2),\n",
              " ('angry', 2),\n",
              " ('ready', 2),\n",
              " ('ee)', 2),\n",
              " ('towers', 2),\n",
              " ('fell', 2),\n",
              " ('ute', 2),\n",
              " ('blue', 2),\n",
              " ('force', 2),\n",
              " (\"week's\", 2),\n",
              " ('offers', 2),\n",
              " ('al-qaeda', 2),\n",
              " ('operates', 2),\n",
              " ('us.', 2),\n",
              " ('used', 2),\n",
              " ('saree', 2),\n",
              " ('hint:', 2),\n",
              " ('supposed', 2),\n",
              " ('benefit', 2),\n",
              " ('fre', 2),\n",
              " ('tt.', 2),\n",
              " ('el)', 2),\n",
              " ('be)', 2),\n",
              " ('ee.', 2),\n",
              " ('maw', 2),\n",
              " ('rx}', 2),\n",
              " ('playing', 2),\n",
              " ('february', 2),\n",
              " ('boa', 2),\n",
              " ('ace', 2),\n",
              " ('toward', 2),\n",
              " ('gamble', 2),\n",
              " ('iraq:', 2),\n",
              " ('occupy', 2),\n",
              " ('gulf', 2),\n",
              " ('baghdad', 2),\n",
              " ('worry', 2),\n",
              " ('breast', 2),\n",
              " ('disease.', 2),\n",
              " ('-or', 2),\n",
              " (\"saddam's\", 2),\n",
              " ('even', 2),\n",
              " ('thought', 2),\n",
              " ('losing', 2),\n",
              " ('politicians', 2),\n",
              " ('cure', 2),\n",
              " ('words', 2),\n",
              " ('gospel', 2),\n",
              " ('laa', 2),\n",
              " ('isaacson', 2),\n",
              " ('eis', 2),\n",
              " ('eek', 2),\n",
              " ...]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bcek5cWK-VOA"
      },
      "source": [
        "### Looks like it's mostly about the US and Geopolitics."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "LkDnXVs3qJ7c"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HwZ4bj5mV7VB",
        "outputId": "cd035ddd-3806-483e-9b59-2ec779eb77a4"
      },
      "source": [
        "# Save all texts in a Pickle File (for quick retreval).\n",
        "# texts = {}\n",
        "# count = 0\n",
        "\n",
        "# for  release_date  in release_dates:\n",
        "#     print(release_date)\n",
        "#     for  file_  in os.listdir(release_date):\n",
        "#         count += 1\n",
        "#         texts[os.path.join(release_date, file_)]  =  (count, get_magazine_text(release_date))\n",
        "\n",
        "\n",
        "# import pickle\n",
        "# with open('time_magazine_texts.pickle', 'wb') as f:\n",
        "#     pickle.dump(texts, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "\n",
        "# with open('time_magazine_texts.pickle', 'rb') as f:\n",
        "#     texts = pickle.load(f)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2000/2000-01-01\n",
            "2000/2000-01-17\n",
            "2000/2000-01-24\n",
            "2000/2000-01-31\n",
            "2000/2000-02-07\n",
            "2000/2000-02-14\n",
            "2000/2000-02-21\n",
            "2000/2000-02-28\n",
            "2000/2000-03-06\n",
            "2000/2000-03-13\n",
            "2000/2000-03-20\n",
            "2000/2000-03-27\n",
            "2000/2000-04-03\n",
            "2000/2000-04-10\n",
            "2000/2000-04-17\n",
            "2000/2000-04-24\n",
            "2000/2000-05-01\n",
            "2000/2000-05-08\n",
            "2000/2000-05-15\n",
            "2000/2000-05-22\n",
            "2000/2000-05-29\n",
            "2000/2000-06-05\n",
            "2000/2000-06-12\n",
            "2000/2000-06-19\n",
            "2000/2000-06-26\n",
            "2000/2000-07-03\n",
            "2000/2000-07-10\n",
            "2000/2000-07-17\n",
            "2000/2000-07-24\n",
            "2000/2000-07-31\n",
            "2000/2000-08-07\n",
            "2000/2000-08-14\n",
            "2000/2000-08-21\n",
            "2000/2000-08-28\n",
            "2000/2000-09-04\n",
            "2000/2000-09-11\n",
            "2000/2000-09-18\n",
            "2000/2000-09-25\n",
            "2000/2000-10-02\n",
            "2000/2000-10-09\n",
            "2000/2000-10-16\n",
            "2000/2000-10-23\n",
            "2000/2000-10-30\n",
            "2000/2000-11-06\n",
            "2000/2000-11-13\n",
            "2000/2000-11-20\n",
            "2000/2000-11-27\n",
            "2000/2000-12-04\n",
            "2000/2000-12-11\n",
            "2000/2000-12-18\n",
            "2000/2000-12-25\n",
            "2001/2001-01-08\n",
            "2001/2001-01-15\n",
            "2001/2001-01-22\n",
            "2001/2001-01-29\n",
            "2001/2001-02-05\n",
            "2001/2001-02-12\n",
            "2001/2001-02-19\n",
            "2001/2001-02-26\n",
            "2001/2001-03-05\n",
            "2001/2001-03-12\n",
            "2001/2001-03-19\n",
            "2001/2001-03-26\n",
            "2001/2001-04-02\n",
            "2001/2001-04-09\n",
            "2001/2001-04-16\n",
            "2001/2001-04-23\n",
            "2001/2001-04-30\n",
            "2001/2001-05-07\n",
            "2001/2001-05-14\n",
            "2001/2001-05-21\n",
            "2001/2001-05-28\n",
            "2001/2001-06-04\n",
            "2001/2001-06-11\n",
            "2001/2001-06-18\n",
            "2001/2001-06-25\n",
            "2001/2001-07-02\n",
            "2001/2001-07-09\n",
            "2001/2001-07-16\n",
            "2001/2001-07-23\n",
            "2001/2001-07-30\n",
            "2001/2001-08-06\n",
            "2001/2001-08-20\n",
            "2001/2001-08-27\n",
            "2001/2001-09-03\n",
            "2001/2001-09-10\n",
            "2001/2001-09-17\n",
            "2001/2001-09-24\n",
            "2001/2001-10-01\n",
            "2001/2001-10-08\n",
            "2001/2001-10-15\n",
            "2001/2001-10-22\n",
            "2001/2001-10-29\n",
            "2001/2001-11-05\n",
            "2001/2001-11-12\n",
            "2001/2001-11-19\n",
            "2001/2001-11-26\n",
            "2001/2001-12-03\n",
            "2001/2001-12-10\n",
            "2001/2001-12-17\n",
            "2001/2001-12-24\n",
            "2001/2001-12-31\n",
            "2002/2002-01-14\n",
            "2002/2002-01-21\n",
            "2002/2002-01-28\n",
            "2002/2002-02-04\n",
            "2002/2002-02-11\n",
            "2002/2002-02-18\n",
            "2002/2002-02-25\n",
            "2002/2002-03-04\n",
            "2002/2002-03-11\n",
            "2002/2002-03-18\n",
            "2002/2002-03-25\n",
            "2002/2002-04-01\n",
            "2002/2002-04-08\n",
            "2002/2002-04-15\n",
            "2002/2002-04-22\n",
            "2002/2002-04-29\n",
            "2002/2002-05-06\n",
            "2002/2002-05-13\n",
            "2002/2002-05-20\n",
            "2002/2002-05-27\n",
            "2002/2002-06-03\n",
            "2002/2002-06-10\n",
            "2002/2002-06-17\n",
            "2002/2002-06-24\n",
            "2002/2002-07-01\n",
            "2002/2002-07-08\n",
            "2002/2002-07-15\n",
            "2002/2002-07-22\n",
            "2002/2002-07-29\n",
            "2002/2002-08-05\n",
            "2002/2002-08-12\n",
            "2002/2002-08-19\n",
            "2002/2002-08-26\n",
            "2002/2002-09-02\n",
            "2002/2002-09-09\n",
            "2002/2002-09-16\n",
            "2002/2002-09-23\n",
            "2002/2002-09-30\n",
            "2002/2002-10-07\n",
            "2002/2002-10-14\n",
            "2002/2002-10-21\n",
            "2002/2002-10-28\n",
            "2002/2002-11-04\n",
            "2002/2002-11-11\n",
            "2002/2002-11-18\n",
            "2002/2002-11-25\n",
            "2002/2002-12-02\n",
            "2002/2002-12-09\n",
            "2002/2002-12-16\n",
            "2002/2002-12-18\n",
            "2002/2002-12-23\n",
            "2002/2002-12-30\n",
            "2003/2003-01-13\n",
            "2003/2003-01-20\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cYeG9KN1-qmp"
      },
      "source": [
        "# Let's try to find some topics and their dominant terms"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XUJzBBkgCIM-",
        "outputId": "6407a37f-684f-4a67-9b2f-3dacc47e8d23"
      },
      "source": [
        "len(release_dates)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "652"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QTbp9tUrCIPv"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ckHX0JP4uls"
      },
      "source": [
        "# Let's see for a year first."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8LEx-hq8BSNH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "27e76475-0c8a-4014-9ec7-b43b8f452f27"
      },
      "source": [
        "%%time\n",
        "l = [get_magazine_text(release_date)   for release_date in release_dates[:1]]\n",
        "pages = [item  for sublist in l  for item in sublist]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 182 ms, sys: 34.2 ms, total: 217 ms\n",
            "Wall time: 21.8 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HEV90fdKAjeX"
      },
      "source": [
        "n_components = 5    # Number of topics"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 878
        },
        "id": "usvsVV5BqJ-F",
        "outputId": "5d39dea7-d80e-4f27-fa90-a7ca92169635"
      },
      "source": [
        "tf_vectorizer = CountVectorizer(strip_accents = 'unicode',\n",
        "                                stop_words = 'english',\n",
        "                                lowercase = True,\n",
        "                                token_pattern = r'\\b[a-zA-Z]{3,}\\b',\n",
        "                                max_df = 0.5, \n",
        "                                min_df = 10)\n",
        "dtm_tf = tf_vectorizer.fit_transform(pages)\n",
        "print(dtm_tf.shape)\n",
        "\n",
        "\n",
        "# for TF DTM\n",
        "lda_tf = LatentDirichletAllocation(n_components=n_components, random_state=0)\n",
        "lda_tf.fit(dtm_tf)\n",
        "\n",
        "\n",
        "pyLDAvis.sklearn.prepare(lda_tf, dtm_tf, tf_vectorizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(126, 173)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.jsdelivr.net/gh/bmabey/pyLDAvis@3.3.1/pyLDAvis/js/ldavis.v1.0.0.css\">\n",
              "\n",
              "\n",
              "<div id=\"ldavis_el2931401918529353766921554949\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "\n",
              "var ldavis_el2931401918529353766921554949_data = {\"mdsDat\": {\"x\": [-0.1563914673131088, 0.16220158948818483, 0.06163784198656425, 0.12306244121596824, -0.19051040537760827], \"y\": [0.20374172813450042, -0.0005063629817602388, 0.0026035207545950585, -0.023692498830515312, -0.1821463870768199], \"topics\": [1, 2, 3, 4, 5], \"cluster\": [1, 1, 1, 1, 1], \"Freq\": [33.89203277456014, 26.958816209597746, 17.014650317774063, 14.583641186575308, 7.5508595114927415]}, \"tinfo\": {\"Term\": [\"com\", \"doctor\", \"new\", \"www\", \"taking\", \"yeltsin\", \"eee\", \"people\", \"time\", \"effects\", \"world\", \"free\", \"symptoms\", \"visit\", \"heart\", \"right\", \"way\", \"best\", \"year\", \"good\", \"like\", \"american\", \"years\", \"blood\", \"patients\", \"city\", \"help\", \"rights\", \"america\", \"information\", \"doctor\", \"effects\", \"symptoms\", \"patients\", \"taking\", \"pregnant\", \"medication\", \"blood\", \"patient\", \"nausea\", \"taken\", \"disease\", \"treatment\", \"used\", \"shown\", \"reported\", \"information\", \"headache\", \"problems\", \"women\", \"possible\", \"use\", \"health\", \"ask\", \"following\", \"usually\", \"medicine\", \"tell\", \"heart\", \"body\", \"important\", \"help\", \"need\", \"including\", \"high\", \"day\", \"time\", \"right\", \"yeltsin\", \"washington\", \"moscow\", \"city\", \"york\", \"president\", \"millennium\", \"new\", \"said\", \"head\", \"century\", \"later\", \"country\", \"party\", \"little\", \"office\", \"send\", \"don\", \"old\", \"let\", \"january\", \"year\", \"service\", \"best\", \"early\", \"fact\", \"says\", \"turn\", \"end\", \"black\", \"time\", \"hours\", \"times\", \"day\", \"world\", \"just\", \"man\", \"life\", \"family\", \"did\", \"like\", \"good\", \"won\", \"live\", \"want\", \"work\", \"past\", \"say\", \"program\", \"know\", \"people\", \"called\", \"look\", \"trying\", \"ago\", \"young\", \"maybe\", \"generally\", \"computer\", \"person\", \"american\", \"years\", \"times\", \"man\", \"child\", \"far\", \"month\", \"great\", \"america\", \"today\", \"black\", \"age\", \"like\", \"right\", \"long\", \"make\", \"children\", \"time\", \"report\", \"year\", \"just\", \"eee\", \"kind\", \"simply\", \"internet\", \"making\", \"america\", \"world\", \"come\", \"today\", \"great\", \"doesn\", \"set\", \"says\", \"american\", \"history\", \"money\", \"turn\", \"thing\", \"days\", \"potential\", \"story\", \"seen\", \"start\", \"better\", \"free\", \"old\", \"years\", \"business\", \"know\", \"higher\", \"like\", \"just\", \"time\", \"year\", \"new\", \"people\", \"january\", \"children\", \"make\", \"ago\", \"millennium\", \"don\", \"age\", \"com\", \"www\", \"feel\", \"rights\", \"complete\", \"free\", \"visit\", \"home\", \"way\", \"business\", \"getting\", \"maybe\", \"best\", \"reserved\", \"took\", \"away\", \"experience\", \"days\", \"family\", \"month\", \"right\", \"life\", \"heart\", \"office\", \"help\", \"things\", \"members\", \"headache\", \"people\", \"contains\", \"don\", \"eee\", \"america\", \"american\", \"use\", \"just\"], \"Freq\": [69.0, 100.0, 121.0, 30.0, 69.0, 44.0, 37.0, 69.0, 177.0, 52.0, 55.0, 30.0, 41.0, 25.0, 38.0, 53.0, 31.0, 31.0, 88.0, 25.0, 67.0, 36.0, 50.0, 38.0, 34.0, 34.0, 41.0, 17.0, 27.0, 55.0, 99.62729543672958, 51.86018240753197, 41.11659458103828, 33.334155640608586, 67.14979151154134, 18.716297402896316, 16.765239829164003, 37.19756816552657, 15.791483717154822, 14.812348031057876, 17.73168306316614, 20.1970981167456, 33.650890459355786, 36.27879774622488, 15.442839847626866, 19.098264907544454, 46.71718075224109, 12.60002146715219, 21.22280540624864, 33.03495070920384, 21.053069866677593, 41.71826920695142, 23.461538023196326, 14.135287711409514, 11.605863403739129, 9.394238994909013, 15.103496604845915, 16.429204592425172, 28.399732116109885, 16.225626122414226, 15.25156879734447, 27.60186528218522, 19.756870591270296, 16.882031723530158, 19.697271338394714, 21.957426121625264, 25.948293501381297, 15.50057894135314, 44.18974433694124, 19.30725219216367, 17.40657584386336, 31.224392936843316, 20.35178080813825, 17.803820063252502, 24.232958055461975, 90.08669628469406, 15.558909538967097, 9.225731171212765, 32.248827313811276, 10.522351788293925, 11.308567530616651, 9.211862271067268, 11.77029985856285, 9.551042072576974, 7.509783888267179, 21.698091663390326, 17.30864428637522, 7.784034262831303, 21.427934932090235, 52.70667698162096, 8.485897474822831, 17.787117755505133, 8.462741615928328, 9.058158082010696, 9.58660456894465, 6.5601076032772365, 8.526007628705512, 6.497764522447105, 89.65475761193663, 15.492823388126087, 12.206031328909804, 20.95651226435015, 22.201341440277243, 15.795516517476342, 12.20997453264466, 12.445967879444183, 10.323773522571267, 10.722899839284137, 10.329386920957202, 21.26573019544405, 9.659148680805547, 8.847736684800505, 13.801602872935725, 10.275479916557627, 11.544055038883485, 9.820195148872205, 9.829462260024817, 16.783212625736386, 34.40406734369884, 11.379349335086784, 7.469064037024876, 5.124210830523419, 11.365346860110453, 7.7416850733061, 6.383809955015796, 9.167713735245249, 8.813541822701414, 5.309536425453686, 16.03748451228672, 21.69992571073859, 10.375582444896068, 13.191756651727106, 6.224726543152448, 6.82945788860227, 4.629298835287551, 4.958540227291073, 10.200545560098748, 5.485601711916775, 4.470526981638596, 9.262819741592615, 23.548556617600127, 18.634008812281646, 11.207842595802397, 10.966841748417146, 10.377989566126258, 26.350299272643078, 7.161488991778939, 10.006966365050753, 7.618218898055183, 31.40457336128009, 6.347841253564515, 6.312585879063929, 5.9819607591452195, 5.862822989465887, 13.928045450797404, 26.869068473889577, 6.109794644849291, 6.8355125217660495, 5.929744851581164, 6.758928521085303, 5.417409989168293, 7.823704783343208, 15.439624599995053, 6.0978726069842235, 10.44223404879611, 5.173189644072767, 5.033026883647705, 8.460790484186324, 5.722603633891602, 6.007545106837152, 5.27558569217618, 6.294651086625953, 6.8569382870101645, 10.09242655295014, 8.686076020093592, 16.14134951401807, 3.880828457971383, 10.492800794191982, 4.154573862423134, 19.917405680551052, 12.048548369014926, 35.03876861667818, 19.413928233084132, 22.76681368610748, 14.873862392089563, 9.128023352133896, 8.732562169089434, 8.20618183685717, 7.050064866694802, 7.533084169538536, 6.624768054722053, 6.124752441643595, 63.25515220430332, 25.034020694254608, 7.578170999485159, 8.3024935809214, 5.971454681616853, 12.230865734793571, 10.395699494697249, 7.140130013476498, 12.146586861350006, 4.213887177268613, 4.491555957759017, 4.176460058445952, 8.810581559472885, 3.0752779327072473, 4.592069435176083, 4.494780443555388, 3.6370907548244706, 4.60305488269211, 5.417470407688442, 2.6388597244105614, 10.47863239160296, 6.55936926559801, 7.495785323792243, 2.4145454649383997, 6.977155521340129, 2.8046346423116, 2.0019596873354355, 2.336867767288088, 10.40378613758976, 2.2494917244786414, 3.6115503310438726, 3.8063473241764267, 3.083394724174229, 3.2648791631294913, 3.229740350858115, 3.1346031416818545], \"Total\": [69.0, 100.0, 121.0, 30.0, 69.0, 44.0, 37.0, 69.0, 177.0, 52.0, 55.0, 30.0, 41.0, 25.0, 38.0, 53.0, 31.0, 31.0, 88.0, 25.0, 67.0, 36.0, 50.0, 38.0, 34.0, 34.0, 41.0, 17.0, 27.0, 55.0, 100.39222639981168, 52.624788003752684, 41.90094386015678, 34.10256020668162, 69.16010398950577, 19.48006512025925, 17.53035934568523, 38.95788026364857, 16.555525397239833, 15.580552238254088, 19.43921766816512, 22.36282112534553, 37.85380098107227, 40.93832186236904, 17.475633715532386, 22.356860851578183, 55.4921943788456, 15.516478518304776, 26.257162262147943, 40.94799689294938, 26.26805671573611, 52.44499748991136, 30.15993936396384, 18.437437701243873, 15.529053525804434, 12.609267787540995, 20.33187545473767, 22.420428012161597, 38.765239125480136, 22.255268269720517, 21.236579780496072, 41.53668075278676, 29.15466538642307, 25.100789016201947, 36.845920937326056, 49.07925617646181, 177.18642249231303, 53.15843412496065, 44.95909326288572, 20.088734253421116, 18.17580433397101, 34.48661424549916, 23.973884769127807, 20.988203813415563, 32.349358318372076, 121.10657825798219, 21.04787900150158, 12.490895189247135, 43.98370739477271, 14.414563702495586, 16.179152811487047, 13.40691059950683, 17.316335885347875, 14.36028488751889, 11.505186740744392, 33.30986230351553, 26.58272385248687, 12.428173788380843, 34.293082522419226, 88.80184765586999, 14.337112901528528, 31.5607620101568, 15.30868779762644, 16.397107775002446, 17.99432894707282, 12.31809252362058, 16.338435259694183, 12.548386604615231, 177.18642249231303, 32.789119984551526, 25.13023498969206, 49.07925617646181, 55.93701519092854, 42.97298231872707, 31.962680928877138, 33.56323991728963, 24.851497668823708, 29.791904644474315, 67.24410776223561, 25.362387521638485, 12.591130735871161, 13.566114996687064, 21.47031180160408, 18.39778633000685, 21.24205196833428, 19.381322877358, 19.472023006711574, 33.69357678277063, 69.19361132573893, 23.24471195709216, 15.43428863200766, 10.602305577833398, 23.995098278961258, 16.348302585641527, 13.485726185160775, 19.478519791203773, 19.230034239288265, 11.61010948324654, 36.3223749033188, 50.92117732196245, 25.13023498969206, 31.962680928877138, 15.456538396018429, 18.266203929270922, 12.44754612114985, 13.360204484354451, 27.60185625865898, 15.26330388640093, 12.548386604615231, 26.017614709996415, 67.24410776223561, 53.15843412496065, 32.86329599293593, 32.73075693935235, 33.67691921631979, 177.18642249231303, 20.27825948853443, 88.80184765586999, 42.97298231872707, 37.51227327827588, 10.446263255814024, 11.403626657244851, 11.385482336484781, 11.395737018641087, 27.60185625865898, 55.93701519092854, 13.352857173755943, 15.26330388640093, 13.360204484354451, 15.243650769650001, 12.331717380891142, 17.99432894707282, 36.3223749033188, 14.36298408201267, 24.73698051017424, 12.31809252362058, 12.377286263631529, 20.864720265828844, 15.340454504218657, 16.243802129840176, 14.301918162945446, 17.26056061513335, 20.061705735440203, 30.39268033387673, 26.58272385248687, 50.92117732196245, 12.387009351991964, 33.69357678277063, 13.463655250058473, 67.24410776223561, 42.97298231872707, 177.18642249231303, 88.80184765586999, 121.10657825798219, 69.19361132573893, 34.293082522419226, 33.67691921631979, 32.73075693935235, 23.995098278961258, 32.349358318372076, 33.30986230351553, 26.017614709996415, 69.18122431248695, 30.401515625417083, 15.361664792884902, 17.206234266696164, 12.483242810116035, 30.39268033387673, 25.92019102865769, 18.231636024948845, 31.53967361325219, 12.387009351991964, 14.320304434666381, 13.485726185160775, 31.5607620101568, 11.520504704736794, 17.25755739525351, 18.31815207477618, 16.459903484145592, 20.864720265828844, 24.851497668823708, 12.44754612114985, 53.15843412496065, 33.56323991728963, 38.765239125480136, 14.36028488751889, 41.53668075278676, 17.19389081489185, 12.526957625011676, 15.516478518304776, 69.19361132573893, 15.448077895714237, 33.30986230351553, 37.51227327827588, 27.60185625865898, 36.3223749033188, 52.44499748991136, 42.97298231872707], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\"], \"logprob\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -2.7048, -3.3577, -3.5898, -3.7997, -3.0993, -4.3768, -4.4869, -3.69, -4.5468, -4.6108, -4.4309, -4.3007, -3.7902, -3.715, -4.5691, -4.3566, -3.4621, -4.7725, -4.2512, -3.8087, -4.2592, -3.5753, -4.1509, -4.6576, -4.8547, -5.0661, -4.5913, -4.5072, -3.9599, -4.5196, -4.5816, -3.9883, -4.3227, -4.48, -4.3258, -4.2171, -4.0501, -4.5654, -3.2889, -4.1169, -4.2205, -3.6362, -4.0642, -4.1979, -3.8896, -2.5766, -4.3327, -4.8554, -3.6039, -4.7239, -4.6518, -4.8569, -4.6118, -4.8207, -5.0612, -4.0001, -4.2262, -5.0253, -4.0127, -3.1126, -4.939, -4.1989, -4.9417, -4.8737, -4.817, -5.1964, -4.9342, -5.2059, -2.5814, -4.337, -4.5754, -4.0349, -3.9772, -4.3176, -4.5751, -4.556, -4.7429, -4.705, -4.7424, -3.56, -4.3492, -4.437, -3.9923, -4.2874, -4.171, -4.3327, -4.3317, -3.7967, -3.079, -4.1853, -4.6064, -4.9832, -4.1866, -4.5705, -4.7634, -4.4014, -4.4408, -4.9476, -3.8422, -3.5398, -4.2777, -4.0375, -4.7886, -4.6959, -5.0847, -5.016, -4.2947, -4.915, -5.1196, -4.3911, -3.4581, -3.6921, -4.2005, -4.2223, -4.2774, -3.3456, -4.6484, -4.3138, -4.5866, -3.016, -4.6148, -4.6204, -4.6742, -4.6943, -3.829, -3.172, -4.6531, -4.5408, -4.683, -4.5521, -4.7733, -4.4058, -3.726, -4.655, -4.1171, -4.8195, -4.8469, -4.3275, -4.7185, -4.6699, -4.7999, -4.6233, -4.5377, -4.1512, -4.3012, -3.6816, -5.1069, -4.1123, -5.0387, -3.4714, -3.974, -2.9065, -3.497, -3.3376, -3.7633, -4.2516, -4.2959, -4.3581, -4.5099, -4.4436, -4.5721, -4.6506, -1.6575, -2.5845, -3.7794, -3.6882, -4.0177, -3.3008, -3.4633, -3.839, -3.3077, -4.3663, -4.3025, -4.3753, -3.6288, -4.6813, -4.2804, -4.3018, -4.5135, -4.278, -4.1151, -4.8344, -3.4554, -3.9238, -3.7904, -4.9232, -3.8621, -4.7734, -5.1106, -4.9559, -3.4625, -4.994, -4.5206, -4.468, -4.6787, -4.6215, -4.6323, -4.6622], \"loglift\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 1.0743, 1.0674, 1.0631, 1.0592, 1.0525, 1.042, 1.0374, 1.0358, 1.0347, 1.0314, 0.9901, 0.9801, 0.9643, 0.9612, 0.9583, 0.9245, 0.9099, 0.8738, 0.8691, 0.8673, 0.8607, 0.8532, 0.8308, 0.8163, 0.7908, 0.7877, 0.7847, 0.7711, 0.7708, 0.766, 0.7509, 0.6733, 0.6929, 0.6853, 0.4557, 0.2777, -0.8391, -0.1504, 1.2936, 1.2712, 1.2676, 1.2115, 1.1471, 1.1463, 1.022, 1.015, 1.0087, 1.0079, 1.0005, 0.9961, 0.9527, 0.9356, 0.9248, 0.903, 0.8843, 0.8822, 0.8818, 0.843, 0.8406, 0.7892, 0.7864, 0.7374, 0.7181, 0.7174, 0.6812, 0.6808, 0.6605, 0.6527, 0.6296, 0.5611, 0.5887, 0.4599, 0.3868, 0.31, 0.3485, 0.3188, 0.4324, 0.289, -0.5625, 1.5949, 1.506, 1.3437, 1.3292, 1.1886, 1.1613, 1.0912, 1.0875, 1.0742, 1.0724, 1.0568, 1.0453, 1.044, 1.0238, 1.0236, 1.0232, 1.0175, 0.9909, 0.9887, 0.9536, 0.9181, 0.8865, 0.8861, 0.8616, 0.7873, 0.782, 0.7799, 0.7757, 0.7478, 0.739, 0.7383, 0.7218, 0.7228, 0.6954, 0.6777, 0.594, -0.1346, 0.7303, -0.412, 0.0411, 1.7476, 1.4271, 1.3339, 1.2817, 1.2607, 1.2413, 1.192, 1.1434, 1.1219, 1.113, 1.112, 1.1027, 1.0924, 1.0698, 1.0686, 1.0628, 1.0577, 1.0254, 1.0227, 0.9392, 0.9306, 0.928, 0.9165, 0.8517, 0.8229, 0.8067, 0.7764, 0.7647, 0.7587, 0.7495, 0.7085, 0.6536, 0.3045, 0.4049, 0.2539, 0.388, 0.6017, 0.5755, 0.5418, 0.7005, 0.468, 0.3102, 0.4788, 2.494, 2.3893, 1.8769, 1.8548, 1.8461, 1.6733, 1.6699, 1.6461, 1.6293, 1.5052, 1.424, 1.4113, 1.3075, 1.2628, 1.2596, 1.1785, 1.0738, 1.0722, 1.0602, 1.0323, 0.9596, 0.951, 0.9403, 0.8006, 0.7996, 0.7702, 0.7498, 0.6904, 0.6888, 0.6567, 0.3618, 0.2955, 0.3917, 0.1743, -0.2039, -0.0346]}, \"token.table\": {\"Topic\": [1, 3, 4, 5, 2, 3, 4, 3, 4, 5, 1, 3, 4, 5, 1, 2, 1, 2, 3, 5, 2, 3, 5, 1, 2, 3, 4, 5, 1, 2, 3, 1, 2, 1, 2, 4, 3, 4, 5, 1, 3, 4, 5, 2, 3, 4, 1, 3, 4, 5, 1, 2, 3, 4, 1, 2, 3, 1, 2, 4, 5, 1, 2, 3, 4, 1, 5, 2, 3, 4, 5, 1, 4, 5, 2, 4, 5, 1, 2, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 1, 5, 1, 1, 2, 3, 4, 1, 2, 4, 5, 1, 2, 4, 2, 3, 4, 5, 1, 1, 2, 3, 4, 1, 3, 5, 1, 2, 3, 1, 2, 3, 4, 5, 2, 3, 4, 1, 2, 3, 5, 1, 2, 3, 5, 1, 3, 4, 5, 1, 2, 3, 1, 2, 3, 4, 5, 1, 2, 3, 2, 3, 4, 2, 3, 1, 5, 1, 2, 3, 1, 3, 5, 1, 2, 4, 5, 1, 2, 3, 4, 1, 3, 4, 1, 3, 4, 2, 3, 5, 1, 2, 3, 5, 1, 4, 5, 1, 2, 4, 1, 2, 3, 5, 2, 3, 4, 2, 3, 4, 1, 2, 3, 4, 5, 1, 3, 4, 1, 2, 3, 4, 1, 2, 3, 1, 2, 4, 1, 2, 3, 4, 5, 1, 2, 3, 4, 1, 2, 3, 2, 3, 5, 1, 2, 3, 4, 5, 2, 3, 4, 1, 2, 3, 4, 1, 2, 4, 1, 2, 3, 2, 3, 5, 1, 1, 3, 4, 1, 2, 3, 5, 2, 4, 2, 3, 4, 5, 2, 3, 4, 5, 2, 1, 1, 3, 4, 1, 2, 3, 4, 1, 2, 5, 2, 4, 2, 3, 4, 1, 2, 3, 4, 1, 1, 1, 2, 3, 4, 5, 1, 2, 3, 1, 2, 3, 1, 4, 1, 2, 4, 1, 3, 4, 1, 3, 4, 5, 1, 2, 3, 4, 1, 2, 1, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 5, 2, 3, 4, 1, 2, 3, 2, 4, 1, 2, 4, 1, 2, 4, 2, 3, 4, 1, 2, 4, 1, 4, 1, 2, 3, 4, 1, 4, 2, 3, 4, 1, 1, 4, 1, 5, 1, 3, 1, 2, 3, 4, 1, 2, 3, 4, 5, 1, 2, 3, 4, 1, 2, 3, 2, 3, 4, 1, 2, 3, 4, 5, 1, 4, 2, 3, 4, 2, 4, 1, 2, 3, 5, 1, 3, 1, 2, 1, 2, 4, 5, 1, 3, 2, 1, 2, 3, 4, 5, 1, 3, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 2, 1, 2, 3, 1, 2, 3, 4], \"Freq\": [0.3074839907182676, 0.345919489558051, 0.2306129930387007, 0.0768709976795669, 0.20837589168717707, 0.45842696171178954, 0.2917262483620479, 0.36229447419366584, 0.5072122638711322, 0.10868834225809976, 0.027531239426434897, 0.44049983082295835, 0.4129685913965234, 0.08259371827930469, 0.7593245995920299, 0.21694988559772282, 0.38213461551282213, 0.21836263743589834, 0.16377197807692376, 0.21836263743589834, 0.5703284348523425, 0.12673965218940944, 0.28516421742617126, 0.09969242029439603, 0.24923105073599006, 0.24923105073599006, 0.34892347103038607, 0.04984621014719801, 0.07969151983508423, 0.4781491190105054, 0.3187660793403369, 0.9497436654561655, 0.0256687477150315, 0.7189308978930106, 0.1347995433549395, 0.08986636223662632, 0.32291894567406276, 0.32291894567406276, 0.32291894567406276, 0.30114376176918983, 0.47322591135158404, 0.12906161218679565, 0.08604107479119709, 0.727542126287496, 0.1364141486789055, 0.1364141486789055, 0.323487696397014, 0.3881852356764169, 0.19409261783820844, 0.0646975392794028, 0.29693927570292755, 0.14846963785146378, 0.29693927570292755, 0.2672453481326348, 0.028996757782057708, 0.8988994912437889, 0.057993515564115417, 0.028909577994256565, 0.04336436699138485, 0.014454788997128282, 0.9106517068190818, 0.07489033897295228, 0.14978067794590455, 0.2995613558918091, 0.4493420338377136, 0.4806443398775985, 0.4806443398775985, 0.26000993746462814, 0.46801788743633066, 0.15600596247877688, 0.10400397498585126, 0.7120626963598968, 0.1294659447927085, 0.1294659447927085, 0.6798872677801834, 0.18542380030368638, 0.061807933434562125, 0.44825455220633725, 0.42787934528786736, 0.08150082767387949, 0.040750413836939746, 0.09585558658437882, 0.19171117316875763, 0.09585558658437882, 0.38342234633751526, 0.23963896646094704, 0.23496315806375717, 0.36922781981447556, 0.23496315806375717, 0.16783082718839798, 0.8943415451878046, 0.08943415451878046, 0.9960930600517849, 0.06560108304179946, 0.19680324912539837, 0.26240433216719783, 0.45920758129259626, 0.03002113881132615, 0.6604650538491753, 0.21014797167928306, 0.1200845552453046, 0.26128954048041836, 0.5225790809608367, 0.19596715536031376, 0.026657941857635176, 0.05331588371527035, 0.8263961975866905, 0.1066317674305407, 0.9881274960441051, 0.24482148604938503, 0.5508483436111163, 0.12241074302469251, 0.12241074302469251, 0.6075369767284565, 0.18226109301853693, 0.2430147906913826, 0.18295909505294158, 0.5488772851588247, 0.2439454600705888, 0.1207170706562088, 0.402390235520696, 0.08047804710413921, 0.1207170706562088, 0.201195117760348, 0.49271321150520114, 0.38322138672626754, 0.10949182477893359, 0.13019422223861732, 0.06509711111930866, 0.26038844447723464, 0.5207768889544693, 0.7727450987312106, 0.06439542489426754, 0.06439542489426754, 0.06439542489426754, 0.19741595456825153, 0.03290265909470859, 0.32902659094708586, 0.39483190913650307, 0.4107088262226521, 0.10267720655566302, 0.4620474295004836, 0.1396618353418821, 0.1396618353418821, 0.20949275301282314, 0.20949275301282314, 0.2793236706837642, 0.0788569293128912, 0.0394284646564456, 0.8279977577853577, 0.1496983075627407, 0.37424576890685174, 0.44909492268822204, 0.7205248193698484, 0.24017493978994944, 0.837818966762588, 0.12889522565578276, 0.7626010026890575, 0.09946969600292055, 0.09946969600292055, 0.7222965892036968, 0.05159261351454977, 0.1805741473009242, 0.674102973385071, 0.1203755309616198, 0.04815021238464792, 0.16852574334626774, 0.5428009258886343, 0.21712037035545373, 0.189980324061022, 0.027140046294431716, 0.5199182443393, 0.14854806981122856, 0.2970961396224571, 0.20887024471168342, 0.3481170745194724, 0.41774048942336683, 0.21939885123453826, 0.32909827685180737, 0.38394798966044197, 0.33547713403661367, 0.4574688191408368, 0.12199168510422315, 0.06099584255211157, 0.7063284274135414, 0.14126568548270826, 0.14126568548270826, 0.6772695467471924, 0.15935754041110411, 0.15935754041110411, 0.8469659656839423, 0.018020552461360473, 0.09010276230680236, 0.036041104922720946, 0.26349344817711406, 0.1756622987847427, 0.5269868963542281, 0.6123683978035854, 0.08748119968622649, 0.2624435990586795, 0.0930817407628898, 0.3723269630515592, 0.1861634815257796, 0.2792452222886694, 0.06981130557216735, 0.09572801063035005, 0.28718403189105013, 0.5743680637821003, 0.1483962368327952, 0.029679247366559042, 0.5045472052315038, 0.2967924736655904, 0.0693742815002351, 0.7631170965025862, 0.1387485630004702, 0.1609246888605476, 0.6436987554421904, 0.1609246888605476, 0.11917800575442827, 0.3575340172632848, 0.20856151007024948, 0.0893835043158212, 0.20856151007024948, 0.19332548876945346, 0.14871191443804113, 0.3569085946512987, 0.29742382887608226, 0.11549787514183582, 0.692987250851015, 0.17324681271275374, 0.2211392134544503, 0.6634176403633509, 0.14742614230296686, 0.365149010086494, 0.121716336695498, 0.3347199259126195, 0.121716336695498, 0.060858168347749, 0.3887448358039118, 0.4535356417712304, 0.06479080596731862, 0.2749707260567279, 0.12220921158076796, 0.33607533184711186, 0.24441842316153592, 0.2632563383213052, 0.1755042255475368, 0.5265126766426104, 0.18771892174348914, 0.3754378434869783, 0.40672433044422646, 0.22245743082794428, 0.44491486165588856, 0.2966099077705924, 0.9697462365016626, 0.7377578144914687, 0.0983677085988625, 0.14755156289829374, 0.07982784247656206, 0.3991392123828103, 0.31931136990624825, 0.15965568495312413, 0.7419003420036852, 0.2473001140012284, 0.20212652865791425, 0.2425518343894971, 0.4042530573158285, 0.12127591719474855, 0.16067423896520167, 0.40168559741300414, 0.2410113584478025, 0.2410113584478025, 0.9353093644514316, 0.962738661032265, 0.6859965544078488, 0.2400987940427471, 0.06859965544078488, 0.016514379555333355, 0.743147079990001, 0.04954313866600006, 0.18991536488633357, 0.13927300298466094, 0.6963650149233047, 0.13927300298466094, 0.6395130948331924, 0.3385657560881607, 0.6712955929109472, 0.14917679842465492, 0.14917679842465492, 0.09415286258509385, 0.1412292938776408, 0.5649171755105632, 0.1883057251701877, 0.9664447135376053, 0.9676692834790275, 0.02890440261290469, 0.11561761045161877, 0.4913748444193798, 0.2167830195967852, 0.14452201306452347, 0.08613183204198085, 0.43065916020990425, 0.43065916020990425, 0.7994500783691306, 0.07613810270182196, 0.07613810270182196, 0.5866840514748101, 0.3911227009832067, 0.9753560823695613, 0.8576246047550998, 0.14293743412584997, 0.799781781075154, 0.11425454015359342, 0.038084846717864475, 0.4108458580416928, 0.5135573225521161, 0.0513557322552116, 0.0513557322552116, 0.246569485059951, 0.246569485059951, 0.3451972790839314, 0.0986277940239804, 0.8498509753286217, 0.13418699610451923, 0.5208105160126386, 0.17360350533754618, 0.2604052580063193, 0.30098704492288963, 0.13168183215376422, 0.3574221158459314, 0.037623380615361204, 0.188116903076806, 0.23247387766550826, 0.2905923470818853, 0.058118469416377064, 0.4649477553310165, 0.7601716067855835, 0.14253217627229692, 0.09502145084819794, 0.10319213051945367, 0.36117245681808785, 0.5159606525972683, 0.5557306432161631, 0.4445845145729304, 0.27968276383817664, 0.34960345479772076, 0.34960345479772076, 0.2607519606244912, 0.6953385616653098, 0.08691732020816373, 0.5579923974196432, 0.2092471490323662, 0.2092471490323662, 0.0810917059735386, 0.405458529867693, 0.405458529867693, 0.8583379718394969, 0.057222531455966466, 0.08769140117058188, 0.08769140117058188, 0.26307420351174565, 0.5261484070234913, 0.5793554579700274, 0.3476132747820164, 0.24624776687299849, 0.3693716503094977, 0.3693716503094977, 0.9784982442599944, 0.925963189839575, 0.051442399435531944, 0.9687666173863252, 0.014459203244572017, 0.713634904352453, 0.22301090761014156, 0.16158630877566812, 0.32317261755133625, 0.16158630877566812, 0.4039657719391703, 0.11632038504442369, 0.29080096261105925, 0.17448057756663554, 0.23264077008884737, 0.17448057756663554, 0.14673810574355928, 0.5079395968046283, 0.14673810574355928, 0.1975320654240221, 0.07958540780937232, 0.47751244685623395, 0.3979270390468616, 0.19654984414435298, 0.32758307357392163, 0.4586163030034903, 0.23178251176497047, 0.2897281397062131, 0.11589125588248524, 0.05794562794124262, 0.2897281397062131, 0.8981924963625383, 0.10566970545441626, 0.37727643017221996, 0.4715955377152749, 0.09431910754305499, 0.5682698020474467, 0.40590700146246195, 0.8008390124926477, 0.09533797767769615, 0.03813519107107846, 0.05720278660661769, 0.8793716586876416, 0.09770796207640461, 0.7137607156612812, 0.23792023855376038, 0.5015395135640411, 0.038579962581849314, 0.07715992516369863, 0.38579962581849314, 0.3260315949150314, 0.6520631898300628, 0.9458037405599258, 0.09511829566744388, 0.2219426898907024, 0.19023659133488777, 0.1268243942232585, 0.38047318266977553, 0.8059002272143402, 0.17094853304546612, 0.7942098457853964, 0.1588419691570793, 0.16306309608058608, 0.10870873072039072, 0.5435436536019537, 0.16306309608058608, 0.017877249913795412, 0.3932994981034991, 0.08938624956897706, 0.4826857476724762, 0.09867929076180203, 0.032893096920600676, 0.032893096920600676, 0.8223274230150169, 0.05630513476900038, 0.596834428551404, 0.11261026953800075, 0.21395951212220143, 0.011261026953800076, 0.1374673636420594, 0.1178291688360509, 0.4320402857321866, 0.31421111689613573, 0.9786674242454648, 0.04171205499776751, 0.8342410999553502, 0.08342410999553503, 0.06116842985756119, 0.18350528957268358, 0.48934743886048954, 0.24467371943024477], \"Term\": [\"age\", \"age\", \"age\", \"age\", \"ago\", \"ago\", \"ago\", \"america\", \"america\", \"america\", \"american\", \"american\", \"american\", \"american\", \"ask\", \"ask\", \"away\", \"away\", \"away\", \"away\", \"best\", \"best\", \"best\", \"better\", \"better\", \"better\", \"better\", \"better\", \"black\", \"black\", \"black\", \"blood\", \"blood\", \"body\", \"body\", \"body\", \"business\", \"business\", \"business\", \"called\", \"called\", \"called\", \"called\", \"century\", \"century\", \"century\", \"child\", \"child\", \"child\", \"child\", \"children\", \"children\", \"children\", \"children\", \"city\", \"city\", \"city\", \"com\", \"com\", \"com\", \"com\", \"come\", \"come\", \"come\", \"come\", \"complete\", \"complete\", \"computer\", \"computer\", \"computer\", \"computer\", \"contains\", \"contains\", \"contains\", \"country\", \"country\", \"country\", \"day\", \"day\", \"day\", \"day\", \"days\", \"days\", \"days\", \"days\", \"days\", \"did\", \"did\", \"did\", \"did\", \"disease\", \"disease\", \"doctor\", \"doesn\", \"doesn\", \"doesn\", \"doesn\", \"don\", \"don\", \"don\", \"don\", \"early\", \"early\", \"early\", \"eee\", \"eee\", \"eee\", \"eee\", \"effects\", \"end\", \"end\", \"end\", \"end\", \"experience\", \"experience\", \"experience\", \"fact\", \"fact\", \"fact\", \"family\", \"family\", \"family\", \"family\", \"family\", \"far\", \"far\", \"far\", \"feel\", \"feel\", \"feel\", \"feel\", \"following\", \"following\", \"following\", \"following\", \"free\", \"free\", \"free\", \"free\", \"generally\", \"generally\", \"generally\", \"getting\", \"getting\", \"getting\", \"getting\", \"getting\", \"good\", \"good\", \"good\", \"great\", \"great\", \"great\", \"head\", \"head\", \"headache\", \"headache\", \"health\", \"health\", \"health\", \"heart\", \"heart\", \"heart\", \"help\", \"help\", \"help\", \"help\", \"high\", \"high\", \"high\", \"high\", \"higher\", \"higher\", \"higher\", \"history\", \"history\", \"history\", \"home\", \"home\", \"home\", \"hours\", \"hours\", \"hours\", \"hours\", \"important\", \"important\", \"important\", \"including\", \"including\", \"including\", \"information\", \"information\", \"information\", \"information\", \"internet\", \"internet\", \"internet\", \"january\", \"january\", \"january\", \"just\", \"just\", \"just\", \"just\", \"just\", \"kind\", \"kind\", \"kind\", \"know\", \"know\", \"know\", \"know\", \"later\", \"later\", \"later\", \"let\", \"let\", \"let\", \"life\", \"life\", \"life\", \"life\", \"life\", \"like\", \"like\", \"like\", \"like\", \"little\", \"little\", \"little\", \"live\", \"live\", \"live\", \"long\", \"long\", \"long\", \"long\", \"long\", \"look\", \"look\", \"look\", \"make\", \"make\", \"make\", \"make\", \"making\", \"making\", \"making\", \"man\", \"man\", \"man\", \"maybe\", \"maybe\", \"maybe\", \"medication\", \"medicine\", \"medicine\", \"medicine\", \"members\", \"members\", \"members\", \"members\", \"millennium\", \"millennium\", \"money\", \"money\", \"money\", \"money\", \"month\", \"month\", \"month\", \"month\", \"moscow\", \"nausea\", \"need\", \"need\", \"need\", \"new\", \"new\", \"new\", \"new\", \"office\", \"office\", \"office\", \"old\", \"old\", \"party\", \"party\", \"party\", \"past\", \"past\", \"past\", \"past\", \"patient\", \"patients\", \"people\", \"people\", \"people\", \"people\", \"people\", \"person\", \"person\", \"person\", \"possible\", \"possible\", \"possible\", \"potential\", \"potential\", \"pregnant\", \"president\", \"president\", \"problems\", \"problems\", \"problems\", \"program\", \"program\", \"program\", \"program\", \"report\", \"report\", \"report\", \"report\", \"reported\", \"reported\", \"reserved\", \"reserved\", \"reserved\", \"right\", \"right\", \"right\", \"right\", \"right\", \"rights\", \"rights\", \"rights\", \"rights\", \"said\", \"said\", \"said\", \"say\", \"say\", \"say\", \"says\", \"says\", \"seen\", \"seen\", \"seen\", \"send\", \"send\", \"send\", \"service\", \"service\", \"service\", \"set\", \"set\", \"set\", \"shown\", \"shown\", \"simply\", \"simply\", \"simply\", \"simply\", \"start\", \"start\", \"story\", \"story\", \"story\", \"symptoms\", \"taken\", \"taken\", \"taking\", \"taking\", \"tell\", \"tell\", \"thing\", \"thing\", \"thing\", \"thing\", \"things\", \"things\", \"things\", \"things\", \"things\", \"time\", \"time\", \"time\", \"time\", \"times\", \"times\", \"times\", \"today\", \"today\", \"today\", \"took\", \"took\", \"took\", \"took\", \"took\", \"treatment\", \"treatment\", \"trying\", \"trying\", \"trying\", \"turn\", \"turn\", \"use\", \"use\", \"use\", \"use\", \"used\", \"used\", \"usually\", \"usually\", \"visit\", \"visit\", \"visit\", \"visit\", \"want\", \"want\", \"washington\", \"way\", \"way\", \"way\", \"way\", \"way\", \"women\", \"women\", \"won\", \"won\", \"work\", \"work\", \"work\", \"work\", \"world\", \"world\", \"world\", \"world\", \"www\", \"www\", \"www\", \"www\", \"year\", \"year\", \"year\", \"year\", \"year\", \"years\", \"years\", \"years\", \"years\", \"yeltsin\", \"york\", \"york\", \"york\", \"young\", \"young\", \"young\", \"young\"]}, \"R\": 30, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [1, 2, 5, 4, 3]};\n",
              "\n",
              "function LDAvis_load_lib(url, callback){\n",
              "  var s = document.createElement('script');\n",
              "  s.src = url;\n",
              "  s.async = true;\n",
              "  s.onreadystatechange = s.onload = callback;\n",
              "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
              "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "}\n",
              "\n",
              "if(typeof(LDAvis) !== \"undefined\"){\n",
              "   // already loaded: just create the visualization\n",
              "   !function(LDAvis){\n",
              "       new LDAvis(\"#\" + \"ldavis_el2931401918529353766921554949\", ldavis_el2931401918529353766921554949_data);\n",
              "   }(LDAvis);\n",
              "}else if(typeof define === \"function\" && define.amd){\n",
              "   // require.js is available: use it to load d3/LDAvis\n",
              "   require.config({paths: {d3: \"https://d3js.org/d3.v5\"}});\n",
              "   require([\"d3\"], function(d3){\n",
              "      window.d3 = d3;\n",
              "      LDAvis_load_lib(\"https://cdn.jsdelivr.net/gh/bmabey/pyLDAvis@3.3.1/pyLDAvis/js/ldavis.v3.0.0.js\", function(){\n",
              "        new LDAvis(\"#\" + \"ldavis_el2931401918529353766921554949\", ldavis_el2931401918529353766921554949_data);\n",
              "      });\n",
              "    });\n",
              "}else{\n",
              "    // require.js not available: dynamically load d3 & LDAvis\n",
              "    LDAvis_load_lib(\"https://d3js.org/d3.v5.js\", function(){\n",
              "         LDAvis_load_lib(\"https://cdn.jsdelivr.net/gh/bmabey/pyLDAvis@3.3.1/pyLDAvis/js/ldavis.v3.0.0.js\", function(){\n",
              "                 new LDAvis(\"#\" + \"ldavis_el2931401918529353766921554949\", ldavis_el2931401918529353766921554949_data);\n",
              "            })\n",
              "         });\n",
              "}\n",
              "</script>"
            ],
            "text/plain": [
              "PreparedData(topic_coordinates=              x         y  topics  cluster       Freq\n",
              "topic                                                \n",
              "0     -0.156391  0.203742       1        1  33.892033\n",
              "1      0.162202 -0.000506       2        1  26.958816\n",
              "4      0.061638  0.002604       3        1  17.014650\n",
              "3      0.123062 -0.023692       4        1  14.583641\n",
              "2     -0.190510 -0.182146       5        1   7.550860, topic_info=         Term        Freq       Total Category  logprob  loglift\n",
              "18        com   69.000000   69.000000  Default  30.0000  30.0000\n",
              "31     doctor  100.000000  100.000000  Default  29.0000  29.0000\n",
              "101       new  121.000000  121.000000  Default  28.0000  28.0000\n",
              "167       www   30.000000   30.000000  Default  27.0000  27.0000\n",
              "142    taking   69.000000   69.000000  Default  26.0000  26.0000\n",
              "..        ...         ...         ...      ...      ...      ...\n",
              "36        eee    3.806347   37.512273   Topic5  -4.4680   0.2955\n",
              "2     america    3.083395   27.601856   Topic5  -4.6787   0.3917\n",
              "3    american    3.264879   36.322375   Topic5  -4.6215   0.1743\n",
              "154       use    3.229740   52.444997   Topic5  -4.6323  -0.2039\n",
              "69       just    3.134603   42.972982   Topic5  -4.6622  -0.0346\n",
              "\n",
              "[228 rows x 6 columns], token_table=      Topic      Freq   Term\n",
              "term                        \n",
              "0         1  0.307484    age\n",
              "0         3  0.345919    age\n",
              "0         4  0.230613    age\n",
              "0         5  0.076871    age\n",
              "1         2  0.208376    ago\n",
              "...     ...       ...    ...\n",
              "171       3  0.083424   york\n",
              "172       1  0.061168  young\n",
              "172       2  0.183505  young\n",
              "172       3  0.489347  young\n",
              "172       4  0.244674  young\n",
              "\n",
              "[458 rows x 3 columns], R=30, lambda_step=0.01, plot_opts={'xlab': 'PC1', 'ylab': 'PC2'}, topic_order=[1, 2, 5, 4, 3])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 878
        },
        "id": "g01NHeccqKAf",
        "outputId": "8590ecc7-ef52-4384-853f-25f55215f9e2"
      },
      "source": [
        "tfidf_vectorizer = TfidfVectorizer(strip_accents = 'unicode',\n",
        "                                   stop_words = 'english',\n",
        "                                   lowercase = True,\n",
        "                                   token_pattern = r'\\b[a-zA-Z]{3,}\\b',\n",
        "                                   max_df = 0.5, \n",
        "                                   min_df = 10)\n",
        "dtm_tfidf = tfidf_vectorizer.fit_transform(pages)\n",
        "print(dtm_tfidf.shape)\n",
        "\n",
        "\n",
        "# for TFIDF DTM\n",
        "lda_tfidf = LatentDirichletAllocation(n_components=n_components, random_state=0)\n",
        "lda_tfidf.fit(dtm_tfidf)\n",
        "\n",
        "\n",
        "pyLDAvis.sklearn.prepare(lda_tfidf, dtm_tfidf, tfidf_vectorizer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(126, 173)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.jsdelivr.net/gh/bmabey/pyLDAvis@3.3.1/pyLDAvis/js/ldavis.v1.0.0.css\">\n",
              "\n",
              "\n",
              "<div id=\"ldavis_el2931401918521182243567067502\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "\n",
              "var ldavis_el2931401918521182243567067502_data = {\"mdsDat\": {\"x\": [0.0959402030150581, -0.13400376032482125, 0.020218527477493534, 0.021652769279085972, -0.003807739446816449], \"y\": [-0.02157299968512011, -0.009077486810616184, -0.03721674399209654, 0.07321884704989731, -0.005351616562064467], \"topics\": [1, 2, 3, 4, 5], \"cluster\": [1, 1, 1, 1, 1], \"Freq\": [63.21903829027325, 19.098245648629284, 7.093560796772791, 6.140157521524396, 4.448997742800292]}, \"tinfo\": {\"Term\": [\"com\", \"yeltsin\", \"www\", \"doctor\", \"best\", \"life\", \"moscow\", \"year\", \"think\", \"president\", \"days\", \"taking\", \"money\", \"washington\", \"things\", \"effects\", \"world\", \"america\", \"eee\", \"people\", \"maybe\", \"said\", \"symptoms\", \"blood\", \"information\", \"home\", \"old\", \"office\", \"free\", \"patients\", \"time\", \"new\", \"eee\", \"city\", \"millennium\", \"world\", \"like\", \"century\", \"years\", \"january\", \"american\", \"just\", \"way\", \"man\", \"york\", \"service\", \"year\", \"better\", \"computer\", \"ago\", \"times\", \"come\", \"got\", \"people\", \"million\", \"far\", \"party\", \"says\", \"good\", \"don\", \"know\", \"make\", \"america\", \"right\", \"com\", \"free\", \"www\", \"did\", \"best\", \"doctor\", \"taking\", \"effects\", \"symptoms\", \"blood\", \"patients\", \"medication\", \"patient\", \"pregnant\", \"disease\", \"taken\", \"nausea\", \"headache\", \"heart\", \"information\", \"ask\", \"reported\", \"experience\", \"treatment\", \"health\", \"following\", \"usually\", \"problems\", \"shown\", \"used\", \"available\", \"tell\", \"mean\", \"medicine\", \"visit\", \"help\", \"use\", \"end\", \"program\", \"women\", \"www\", \"life\", \"maybe\", \"office\", \"com\", \"home\", \"best\", \"complete\", \"getting\", \"live\", \"reserved\", \"business\", \"rights\", \"potential\", \"president\", \"america\", \"family\", \"away\", \"long\", \"hours\", \"free\", \"people\", \"help\", \"don\", \"signs\", \"shown\", \"usually\", \"contains\", \"million\", \"headache\", \"day\", \"just\", \"year\", \"eee\", \"time\", \"yeltsin\", \"moscow\", \"president\", \"washington\", \"said\", \"black\", \"old\", \"best\", \"year\", \"usually\", \"signs\", \"shown\", \"contains\", \"large\", \"headache\", \"nausea\", \"mean\", \"person\", \"common\", \"trying\", \"kind\", \"state\", \"head\", \"send\", \"office\", \"light\", \"following\", \"continue\", \"pregnant\", \"patient\", \"soon\", \"world\", \"eee\", \"end\", \"century\", \"blood\", \"little\", \"possible\", \"later\", \"ask\", \"turn\", \"money\", \"fact\", \"time\", \"taking\", \"new\", \"think\", \"things\", \"days\", \"money\", \"signs\", \"shown\", \"usually\", \"contains\", \"headache\", \"large\", \"nausea\", \"person\", \"kind\", \"send\", \"trying\", \"common\", \"light\", \"following\", \"continue\", \"pregnant\", \"patient\", \"child\", \"disease\", \"week\", \"possible\", \"month\", \"months\", \"room\", \"complete\", \"members\", \"com\", \"eee\", \"mean\", \"reserved\", \"available\", \"president\", \"potential\", \"end\", \"away\", \"black\", \"rights\", \"www\", \"soon\", \"taking\", \"free\", \"office\", \"ask\", \"taken\", \"american\", \"help\", \"night\", \"thing\", \"know\"], \"Freq\": [6.0, 3.0, 4.0, 4.0, 3.0, 3.0, 1.0, 5.0, 2.0, 1.0, 2.0, 3.0, 2.0, 2.0, 2.0, 2.0, 6.0, 3.0, 8.0, 5.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 1.0, 3.0, 2.0, 9.681453802359409, 8.177374273030498, 7.800713447581795, 4.257830249285491, 3.9594572667112025, 6.188589175353916, 4.026448796446758, 4.109567967223725, 3.271494719329614, 2.7481299509759065, 3.190827256664848, 2.9826136756492323, 2.529059035437223, 2.495261933062366, 2.425164067548316, 2.146003119655229, 4.76292089955615, 2.094531246172542, 2.068663695193362, 2.0412821307553743, 2.04563658890312, 2.0015536566003997, 1.9580589050280588, 4.294260849526855, 2.5734279011834853, 1.9218413922047801, 1.8778581793538442, 1.8633454273756822, 2.3358666260605587, 2.565731569111429, 2.3512747773231477, 2.388576023070485, 3.0108456191544524, 2.8933292843324208, 3.907605111673691, 2.3523902642741317, 2.4797705584443746, 2.072163976610468, 2.072943661789253, 3.988327239924792, 2.5183602529507416, 2.2071023945394708, 1.8920050478640023, 1.872866663537663, 1.5623984080996458, 1.3032016781336115, 1.0083980589643393, 1.0066754098116857, 1.029635724530888, 1.1385730714871751, 0.9178207113829288, 0.8988012492450121, 1.3864535651383647, 1.8941627574056887, 1.067630797017521, 0.9538150632840892, 1.081430915785063, 1.0993291762770812, 1.3843828740924353, 0.7981326068827594, 0.6858543811448976, 1.0119057956925646, 0.6360430362178143, 1.1028711103928392, 1.3948924986386415, 0.8177780786032627, 0.9548866070061881, 0.7795797598391, 1.142237297715157, 1.345950201414667, 1.0821849928784006, 0.9452615397253417, 0.9480502628628803, 0.8603773488293904, 1.5587064042512035, 0.9511221986443704, 0.573861525643937, 0.5101796679909542, 1.7946290307724357, 0.515794457329776, 0.8624249190354402, 0.34436230627027464, 0.37697842786798286, 0.3646229246825489, 0.36693495805658116, 0.3842915721452864, 0.3542352115569731, 0.27904034238668396, 0.2719950434712059, 0.49728186020676163, 0.3370895761794708, 0.24109006688879522, 0.2908452335287899, 0.315513513841043, 0.42145008620642027, 0.4959416701773386, 0.2869369461034473, 0.29008454186899796, 0.11238121435801368, 0.11238555111658548, 0.11241718796008893, 0.11238117577938002, 0.2658016757937374, 0.11238862663772717, 0.2060990720498747, 0.2000423115039746, 0.2445090938733632, 0.2718415443800231, 0.11436314941170997, 2.571125911248698, 1.3067276632230855, 1.1880361386014229, 0.5330434482849703, 0.4082084620343865, 0.2816795712722488, 0.3639203795662334, 0.45119764323057093, 0.5541359641077973, 0.11647075951846918, 0.10901313095941298, 0.10678265990631189, 0.10677834446819003, 0.10709242768729484, 0.10678387152630277, 0.10674425977983523, 0.15765648345604938, 0.10739217040994523, 0.10973392808700996, 0.10851812272586947, 0.10721140279991749, 0.11974197788972821, 0.1346996181725481, 0.10705415704107107, 0.14204452762754272, 0.10731444738973216, 0.1067910147532228, 0.10677265208800885, 0.10674177154988743, 0.10674252673300384, 0.11635966627888021, 0.37106788110951827, 0.3041135448429125, 0.13966720976723465, 0.18151105268284962, 0.11825292174001646, 0.11095808955901261, 0.10867979587183635, 0.109275907006662, 0.11055115278453202, 0.10897143182095527, 0.11481966623983618, 0.11140428975562451, 0.12499005544114332, 0.11277519241721723, 0.11106765865195062, 0.6119233284753788, 0.3803625647643988, 0.37400762469463994, 0.36284002344614064, 0.10029984399378719, 0.10030427954010067, 0.10034146831830647, 0.10029913194518474, 0.10030495867669914, 0.10028513941295031, 0.10025522190445098, 0.10027907311383082, 0.10027256536427816, 0.1002834357966452, 0.10027239234707301, 0.10029583397418734, 0.1002922197337744, 0.10031403736391176, 0.10029111434910441, 0.10025204766005101, 0.10025280818850288, 0.10029333328074079, 0.10030059566522817, 0.10028608764335079, 0.1002905625676197, 0.10027800609150292, 0.10029564324295034, 0.10029410623165597, 0.10033271123407193, 0.10028376387455734, 0.27348422488656615, 0.10037370202407375, 0.1003506660841951, 0.10034155691567194, 0.10033314226836408, 0.1003295836019363, 0.10032874767419483, 0.1003259531193534, 0.10032550787848137, 0.10032444874424197, 0.10032368929114494, 0.10032266669616346, 0.10032239965524127, 0.10032187735032626, 0.10032033538756627, 0.10031813376973885, 0.10031685101889166, 0.10031651253956994, 0.10031336660606488, 0.10031302446553507, 0.10031299351365083, 0.1003117347112888, 0.10030989312070239], \"Total\": [6.0, 3.0, 4.0, 4.0, 3.0, 3.0, 1.0, 5.0, 2.0, 1.0, 2.0, 3.0, 2.0, 2.0, 2.0, 2.0, 6.0, 3.0, 8.0, 5.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 1.0, 3.0, 2.0, 10.180428438403968, 8.644329620689414, 8.62142383429712, 4.721350174505246, 4.421971592785151, 6.915472537627395, 4.5386294458411, 4.646674476315923, 3.738362640130412, 3.211088466953595, 3.75608501624448, 3.5344881196023596, 2.99968532218736, 2.961233378130213, 2.8885609596920863, 2.6095008580720767, 5.806999953575849, 2.562345175583004, 2.5310779368886567, 2.504113402200711, 2.5109568656983217, 2.464545852514955, 2.420422810935148, 5.316613951929141, 3.19047106929991, 2.3845712511863417, 2.341343987592385, 2.326789326889158, 2.9203981912187493, 3.20788171164788, 2.9397690887554235, 3.0137703667523614, 3.8583383442954635, 4.006813025268555, 6.808125093272591, 3.6483579788452225, 4.920558727196052, 2.707212977400945, 3.631750717288719, 4.4640904299408195, 3.0195096633070015, 2.6831390344701864, 2.3718574091625193, 2.3639381208108503, 2.040489761201579, 1.7794450344588841, 1.483998167801599, 1.482289631559955, 1.5203830314652833, 1.6933266917976078, 1.3937190020527326, 1.390936191210613, 2.184655139036454, 2.988184553611709, 1.7476235486841378, 1.6641125379600012, 1.8942986193442903, 1.9272290202138367, 2.447065563056413, 1.4551983343198747, 1.290219352327935, 1.9672424642795165, 1.2746572133923233, 2.220201234932957, 2.827005111791236, 1.7478710589468607, 2.0596952654989393, 1.7623438311741038, 2.6007339270463703, 3.1033775908833197, 2.837997404542877, 2.330226727593365, 2.3650665900856285, 2.213128914001977, 4.920558727196052, 3.3534052655001387, 2.13529534886231, 1.9150377980458082, 6.808125093272591, 2.117563279143303, 3.631750717288719, 1.586937251425569, 1.9003647500500007, 1.9060916855212482, 1.980249274005586, 2.246855434269807, 2.3468552354795436, 1.8756643608914594, 1.9131117255883126, 3.8583383442954635, 2.6155573007238, 1.909326720228952, 2.4314275526167752, 2.6904007832889563, 3.6483579788452225, 5.316613951929141, 3.1033775908833197, 3.20788171164788, 1.2551224986318228, 1.2746572133923233, 1.290219352327935, 1.3175714842673374, 3.19047106929991, 1.390936191210613, 3.0480889756845007, 3.5344881196023596, 5.806999953575849, 8.62142383429712, 10.180428438403968, 3.087402219397138, 1.8264564040002738, 1.9131117255883126, 2.3464084021920817, 2.4390284249886656, 1.9921252891446133, 2.6369308993292195, 3.631750717288719, 5.806999953575849, 1.290219352327935, 1.2551224986318228, 1.2746572133923233, 1.3175714842673374, 1.3922221840868805, 1.390936191210613, 1.3937190020527326, 2.0596952654989393, 1.4104954058449017, 1.4443560264300523, 1.4376306556011054, 1.4219115340576565, 1.5912169254413357, 1.7935170035208794, 1.4286263154671288, 1.9150377980458082, 1.4533989316185107, 1.4551983343198747, 1.478750677305947, 1.482289631559955, 1.483998167801599, 1.6396839879875604, 6.915472537627395, 8.62142383429712, 2.330226727593365, 4.646674476315923, 2.3639381208108503, 1.7424812449533482, 1.5484332673944747, 1.6137594184577235, 1.7476235486841378, 1.591794531289897, 2.593616468881171, 2.0468381470529686, 10.180428438403968, 3.0195096633070015, 8.644329620689414, 2.0700198629296027, 2.239802414805793, 2.6328911424321757, 2.593616468881171, 1.2551224986318228, 1.2746572133923233, 1.290219352327935, 1.3175714842673374, 1.390936191210613, 1.3922221840868805, 1.3937190020527326, 1.4104954058449017, 1.4219115340576565, 1.4286263154671288, 1.4376306556011054, 1.4443560264300523, 1.4533989316185107, 1.4551983343198747, 1.478750677305947, 1.482289631559955, 1.483998167801599, 1.5024277956312373, 1.5203830314652833, 1.5247783287824554, 1.5484332673944747, 1.548561617558891, 1.5581117251665395, 1.5703672541869629, 1.586937251425569, 1.5896253478206583, 6.808125093272591, 8.62142383429712, 2.0596952654989393, 1.980249274005586, 2.827005111791236, 1.9131117255883126, 1.8756643608914594, 2.330226727593365, 1.909326720228952, 1.9921252891446133, 2.3468552354795436, 4.920558727196052, 1.6396839879875604, 3.0195096633070015, 3.6483579788452225, 1.9150377980458082, 1.7476235486841378, 1.6933266917976078, 3.75608501624448, 3.1033775908833197, 2.41984499654989, 2.0952791798667354, 2.9397690887554235], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\"], \"logprob\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -3.3203, -3.4892, -3.5363, -4.1418, -4.2144, -3.7678, -4.1977, -4.1772, -4.4053, -4.5796, -4.4303, -4.4977, -4.6627, -4.6761, -4.7046, -4.8269, -4.0297, -4.8512, -4.8636, -4.877, -4.8748, -4.8966, -4.9186, -4.1333, -4.6453, -4.9373, -4.9604, -4.9682, -4.7422, -4.6483, -4.7356, -4.7198, -4.4883, -4.5281, -4.2276, -4.7351, -4.6824, -4.8619, -4.8616, -3.0102, -3.4699, -3.6018, -3.7559, -3.7661, -3.9473, -4.1287, -4.3852, -4.3869, -4.3643, -4.2638, -4.4793, -4.5002, -4.0668, -3.7548, -4.3281, -4.4408, -4.3152, -4.2988, -4.0683, -4.619, -4.7706, -4.3817, -4.846, -4.2956, -4.0607, -4.5947, -4.4397, -4.6425, -4.2605, -4.0964, -4.3145, -4.4498, -4.4469, -4.5439, -2.9593, -3.4532, -3.9585, -4.0761, -2.8183, -4.0652, -3.5511, -4.4692, -4.3787, -4.412, -4.4057, -4.3595, -4.4409, -4.6795, -4.7051, -4.1017, -4.4905, -4.8257, -4.6381, -4.5567, -4.2672, -4.1044, -4.6516, -4.6407, -5.589, -5.5889, -5.5887, -5.589, -4.7281, -5.5889, -4.9825, -5.0123, -4.8116, -4.7057, -5.5715, -2.3144, -2.9913, -3.0865, -3.8879, -4.1548, -4.5258, -4.2696, -4.0546, -3.8491, -5.4089, -5.4751, -5.4957, -5.4958, -5.4928, -5.4957, -5.4961, -5.1061, -5.4901, -5.4685, -5.4796, -5.4917, -5.3812, -5.2635, -5.4932, -5.2104, -5.4908, -5.4957, -5.4958, -5.4961, -5.4961, -5.4099, -4.2502, -4.4491, -5.2273, -4.9652, -5.3937, -5.4574, -5.4781, -5.4727, -5.4611, -5.4755, -5.4232, -5.4534, -5.3383, -5.4411, -5.4564, -3.4278, -3.9032, -3.9201, -3.9504, -5.2362, -5.2362, -5.2358, -5.2362, -5.2362, -5.2363, -5.2366, -5.2364, -5.2365, -5.2364, -5.2365, -5.2362, -5.2363, -5.2361, -5.2363, -5.2367, -5.2367, -5.2363, -5.2362, -5.2363, -5.2363, -5.2364, -5.2362, -5.2363, -5.2359, -5.2364, -4.2331, -5.2355, -5.2357, -5.2358, -5.2359, -5.2359, -5.2359, -5.2359, -5.2359, -5.236, -5.236, -5.236, -5.236, -5.236, -5.236, -5.236, -5.236, -5.236, -5.2361, -5.2361, -5.2361, -5.2361, -5.2361], \"loglift\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.4083, 0.403, 0.3585, 0.3552, 0.3481, 0.3475, 0.3388, 0.3357, 0.3252, 0.3029, 0.2955, 0.2888, 0.2879, 0.2874, 0.2837, 0.263, 0.2604, 0.257, 0.2568, 0.2542, 0.2536, 0.2505, 0.2466, 0.245, 0.2436, 0.2428, 0.238, 0.2364, 0.2352, 0.2352, 0.2352, 0.2261, 0.2105, 0.133, -0.0966, 0.0197, -0.2267, 0.1912, -0.1022, 1.5429, 1.4741, 1.4603, 1.4295, 1.4227, 1.3886, 1.3441, 1.2692, 1.2686, 1.2658, 1.2587, 1.2378, 1.2189, 1.2009, 1.1997, 1.1628, 1.099, 1.095, 1.0942, 1.0859, 1.055, 1.0237, 0.9908, 0.9604, 0.9559, 0.9492, 0.896, 0.8869, 0.8399, 0.8328, 0.8202, 0.6915, 0.7533, 0.7414, 0.7108, 1.4964, 1.3859, 1.332, 1.3233, 1.3127, 1.2337, 1.2083, 1.1181, 1.0284, 0.992, 0.9602, 0.8801, 0.7551, 0.7406, 0.6953, 0.5971, 0.5971, 0.5766, 0.5225, 0.5027, 0.4877, 0.2738, 0.265, 0.2428, 0.2329, 0.2175, 0.2056, 0.1843, 0.1608, 0.1302, -0.0479, -0.2258, -0.5216, -0.8108, -1.8429, 2.6073, 2.4555, 2.3139, 1.3083, 1.0027, 0.8341, 0.8099, 0.7048, 0.4409, 0.3854, 0.3468, 0.3107, 0.2775, 0.2254, 0.2234, 0.221, 0.2204, 0.2151, 0.213, 0.2065, 0.2054, 0.2034, 0.2014, 0.1992, 0.189, 0.1844, 0.1783, 0.1621, 0.1594, 0.1582, 0.1447, -0.1348, -0.5543, -0.0241, -0.4523, -0.2049, 0.0364, 0.1337, 0.0979, 0.0298, 0.1088, -0.3271, -0.1206, -1.6097, -0.4971, -1.5642, 1.8938, 1.3395, 1.1609, 1.1456, 0.5857, 0.5703, 0.5585, 0.5371, 0.483, 0.4819, 0.4805, 0.4688, 0.4606, 0.456, 0.4496, 0.4452, 0.4389, 0.4379, 0.4216, 0.4188, 0.4177, 0.4058, 0.3939, 0.3909, 0.3756, 0.3754, 0.3694, 0.3615, 0.3514, 0.3492, -0.1021, -1.3406, 0.0908, 0.1301, -0.226, 0.1645, 0.1842, -0.0328, 0.1664, 0.1239, -0.0399, -0.7803, 0.3186, -0.292, -0.4812, 0.1633, 0.2548, 0.2864, -0.5103, -0.3195, -0.0707, 0.0733, -0.2653]}, \"token.table\": {\"Topic\": [1, 1, 1, 2, 1, 2, 1, 1, 3, 1, 1, 2, 1, 1, 1, 1, 1, 2, 3, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 1, 1, 2, 1, 2, 2, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1, 3, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 2, 2, 1, 2, 1, 1, 1, 1, 1, 1, 4, 2, 1, 1, 1, 3, 1, 1, 2, 2, 1, 1, 1, 2, 1, 2, 4, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 2, 1, 2, 2, 2, 1, 2, 1, 1, 1, 5, 1, 1, 1, 2, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1, 4, 1, 1, 1, 2, 1, 1, 2, 3, 1, 4, 1, 4, 1], \"Freq\": [0.798685873508094, 0.7775367871600704, 0.798703966237577, 0.5722056107294639, 0.3537312316235551, 0.3537312316235551, 0.5237448307852139, 0.5506985902086085, 0.2753492951043042, 0.7805349642422572, 0.5019764597383249, 0.8460458344459472, 0.890132925107382, 0.8608306909356316, 0.6655893899911877, 0.8472152778668155, 0.5875332702027724, 0.1468833175506931, 0.2937666351013862, 0.8115085373473951, 0.6923500727667911, 0.6923500727667911, 0.6301446381082083, 0.7901771695179456, 0.7589721027971932, 0.6762465203544955, 0.6561488250358131, 0.32807441251790653, 0.7596212269347633, 0.7387671441794346, 0.6577289928290246, 0.8960391960637384, 0.9351965781989224, 0.927920973815831, 0.7453955886393046, 0.42914279033817027, 0.42914279033817027, 0.5278998726959686, 0.9771168291345331, 0.7646553946444005, 0.8387252001822069, 0.6871915507430651, 0.5481918198808549, 0.27409590994042743, 0.5262147700717396, 0.6848381176285259, 0.8263019134360601, 0.5575637131049694, 0.7189402406228581, 0.4086527206696451, 0.4086527206696451, 0.45773814920786615, 0.3222295614100147, 0.3222295614100147, 0.4722409053129064, 0.4722409053129064, 0.7433836670070559, 0.3346513517015998, 0.6693027034031996, 0.9342626436094869, 0.848779200405831, 0.7032786330569645, 0.6803255424550086, 0.71827615694536, 0.6196710541622766, 0.5964086776435931, 0.29820433882179653, 0.6880423387172826, 0.8813233262885859, 0.5738942688170932, 0.5246337348806679, 0.822562036795026, 0.6636205671353785, 0.6753942511828782, 0.4683192891947253, 0.4683192891947253, 0.48550871420183633, 0.48550871420183633, 0.5619729638370609, 0.567426164129268, 0.567426164129268, 0.6290790476957216, 0.9045738797884555, 0.940300016780373, 0.7711240362622912, 0.6457605487964837, 0.6418024996847479, 0.5475082776735415, 0.717504747030897, 0.9254621643363449, 0.8264992191035018, 0.5221829047032103, 0.5221829047032103, 0.7584574933339203, 0.8542102359152315, 0.6738552794046937, 0.9801568417683527, 0.7523585568120089, 0.7089707601004126, 0.6458140761097731, 0.6458140761097731, 0.5331444265032169, 0.6746319873718637, 0.5227086252333141, 0.5083257494475854, 0.5083257494475854, 0.42282107581748657, 0.42282107581748657, 0.6009208975889803, 0.504986929235041, 0.504986929235041, 0.7487247298740439, 0.24957490995801465, 0.42610212376208434, 0.6367937164594896, 0.6367937164594896, 0.8199986435210546, 0.8595535388130455, 0.6999731064543792, 0.7664300986195569, 0.7845246466998282, 0.7967349809202485, 0.6098736142610832, 0.6098736142610832, 0.6284498260491056, 0.8432210099451894, 0.5905534973516637, 0.9935387975259419, 0.5721245825779202, 0.5721245825779202, 0.9545267376384681, 0.4464679533291365, 0.48308715191976304, 0.48308715191976304, 0.9822769307307999, 0.7965091026937177, 0.5188796917810238, 0.5188796917810238, 0.6955889512399677, 0.6282217838690893, 0.35236113972453487, 0.35236113972453487, 0.4504096224548748, 0.4504096224548748, 0.7750620064686722, 0.38450684616387915, 0.38450684616387915, 0.4261832676126507, 0.4261832676126507, 1.000104903607826, 0.6558330356114819, 0.45184896084146803, 0.45184896084146803, 0.8676196698567931, 0.40645790668973214, 0.20322895334486607, 0.40645790668973214, 0.8610297985143065, 0.1722059597028613, 0.8024903651121833, 0.9716906923082393, 0.6923862878120444], \"Term\": [\"ago\", \"america\", \"american\", \"ask\", \"available\", \"available\", \"away\", \"best\", \"best\", \"better\", \"black\", \"blood\", \"business\", \"century\", \"child\", \"city\", \"com\", \"com\", \"com\", \"come\", \"common\", \"common\", \"complete\", \"computer\", \"contains\", \"continue\", \"day\", \"day\", \"days\", \"did\", \"disease\", \"doctor\", \"don\", \"eee\", \"effects\", \"end\", \"end\", \"experience\", \"fact\", \"family\", \"far\", \"following\", \"free\", \"free\", \"getting\", \"good\", \"got\", \"head\", \"headache\", \"health\", \"health\", \"heart\", \"help\", \"help\", \"home\", \"home\", \"hours\", \"information\", \"information\", \"january\", \"just\", \"kind\", \"know\", \"large\", \"later\", \"life\", \"life\", \"light\", \"like\", \"little\", \"live\", \"long\", \"make\", \"man\", \"maybe\", \"maybe\", \"mean\", \"mean\", \"medication\", \"medicine\", \"medicine\", \"members\", \"millennium\", \"million\", \"money\", \"month\", \"months\", \"moscow\", \"nausea\", \"new\", \"night\", \"office\", \"office\", \"old\", \"party\", \"patient\", \"patients\", \"people\", \"person\", \"possible\", \"possible\", \"potential\", \"pregnant\", \"president\", \"problems\", \"problems\", \"program\", \"program\", \"reported\", \"reserved\", \"reserved\", \"right\", \"right\", \"rights\", \"room\", \"room\", \"said\", \"says\", \"send\", \"service\", \"shown\", \"signs\", \"soon\", \"soon\", \"state\", \"symptoms\", \"taken\", \"taking\", \"tell\", \"tell\", \"thing\", \"things\", \"think\", \"think\", \"time\", \"times\", \"treatment\", \"treatment\", \"trying\", \"turn\", \"use\", \"use\", \"used\", \"used\", \"usually\", \"visit\", \"visit\", \"washington\", \"washington\", \"way\", \"week\", \"women\", \"women\", \"world\", \"www\", \"www\", \"www\", \"year\", \"year\", \"years\", \"yeltsin\", \"york\"]}, \"R\": 30, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [4, 1, 3, 2, 5]};\n",
              "\n",
              "function LDAvis_load_lib(url, callback){\n",
              "  var s = document.createElement('script');\n",
              "  s.src = url;\n",
              "  s.async = true;\n",
              "  s.onreadystatechange = s.onload = callback;\n",
              "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
              "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "}\n",
              "\n",
              "if(typeof(LDAvis) !== \"undefined\"){\n",
              "   // already loaded: just create the visualization\n",
              "   !function(LDAvis){\n",
              "       new LDAvis(\"#\" + \"ldavis_el2931401918521182243567067502\", ldavis_el2931401918521182243567067502_data);\n",
              "   }(LDAvis);\n",
              "}else if(typeof define === \"function\" && define.amd){\n",
              "   // require.js is available: use it to load d3/LDAvis\n",
              "   require.config({paths: {d3: \"https://d3js.org/d3.v5\"}});\n",
              "   require([\"d3\"], function(d3){\n",
              "      window.d3 = d3;\n",
              "      LDAvis_load_lib(\"https://cdn.jsdelivr.net/gh/bmabey/pyLDAvis@3.3.1/pyLDAvis/js/ldavis.v3.0.0.js\", function(){\n",
              "        new LDAvis(\"#\" + \"ldavis_el2931401918521182243567067502\", ldavis_el2931401918521182243567067502_data);\n",
              "      });\n",
              "    });\n",
              "}else{\n",
              "    // require.js not available: dynamically load d3 & LDAvis\n",
              "    LDAvis_load_lib(\"https://d3js.org/d3.v5.js\", function(){\n",
              "         LDAvis_load_lib(\"https://cdn.jsdelivr.net/gh/bmabey/pyLDAvis@3.3.1/pyLDAvis/js/ldavis.v3.0.0.js\", function(){\n",
              "                 new LDAvis(\"#\" + \"ldavis_el2931401918521182243567067502\", ldavis_el2931401918521182243567067502_data);\n",
              "            })\n",
              "         });\n",
              "}\n",
              "</script>"
            ],
            "text/plain": [
              "PreparedData(topic_coordinates=              x         y  topics  cluster       Freq\n",
              "topic                                                \n",
              "3      0.095940 -0.021573       1        1  63.219038\n",
              "0     -0.134004 -0.009077       2        1  19.098246\n",
              "2      0.020219 -0.037217       3        1   7.093561\n",
              "1      0.021653  0.073219       4        1   6.140158\n",
              "4     -0.003808 -0.005352       5        1   4.448998, topic_info=         Term      Freq     Total Category  logprob  loglift\n",
              "18        com  6.000000  6.000000  Default  30.0000  30.0000\n",
              "170   yeltsin  3.000000  3.000000  Default  29.0000  29.0000\n",
              "167       www  4.000000  4.000000  Default  28.0000  28.0000\n",
              "31     doctor  4.000000  4.000000  Default  27.0000  27.0000\n",
              "7        best  3.000000  3.000000  Default  26.0000  26.0000\n",
              "..        ...       ...       ...      ...      ...      ...\n",
              "3    american  0.100313  3.756085   Topic5  -5.2361  -0.5103\n",
              "57       help  0.100313  3.103378   Topic5  -5.2361  -0.3195\n",
              "102     night  0.100313  2.419845   Topic5  -5.2361  -0.0707\n",
              "144     thing  0.100312  2.095279   Topic5  -5.2361   0.0733\n",
              "71       know  0.100310  2.939769   Topic5  -5.2361  -0.2653\n",
              "\n",
              "[238 rows x 6 columns], token_table=      Topic      Freq       Term\n",
              "term                            \n",
              "1         1  0.798686        ago\n",
              "2         1  0.777537    america\n",
              "3         1  0.798704   american\n",
              "4         2  0.572206        ask\n",
              "5         1  0.353731  available\n",
              "...     ...       ...        ...\n",
              "168       1  0.861030       year\n",
              "168       4  0.172206       year\n",
              "169       1  0.802490      years\n",
              "170       4  0.971691    yeltsin\n",
              "171       1  0.692386       york\n",
              "\n",
              "[162 rows x 3 columns], R=30, lambda_step=0.01, plot_opts={'xlab': 'PC1', 'ylab': 'PC2'}, topic_order=[4, 1, 3, 2, 5])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iB-Ayy2FqKDF"
      },
      "source": [
        "# TODO: For all the issues, and interpret "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BtNwJGCHqKF2"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TO0-N0HHDY2c"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Op1Q7L4EDY5O"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}