Text_Encoding_Workshop_5Apr2019.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text_Encoding_Workshop-5Apr2019.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "_St7nPQ6lczd",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PwLSxCYpleZv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "text = '''Mr. and Mrs. Dursley, of number four, Privet Drive, were proud to say that they were perfectly normal, thank you very much.\n",
        "They were the last people you’d expect to be involved in anything strange or mysterious, because they just didn’t hold with such nonsense.'''"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AEVFrbUdmQ_g",
        "colab_type": "code",
        "outputId": "5a82c74e-f743-4c40-c21f-49b03c2cc6c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "ascii_chars = [ord(char) for char in text]\n",
        "ascii_chars[:5]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[77, 114, 46, 32, 97]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "metadata": {
        "id": "T3W6379XmoXh",
        "colab_type": "code",
        "outputId": "bb3f004a-78e9-4139-dea0-36d7c8b4d774",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "len(text)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "262"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "metadata": {
        "id": "M57OFJbooTXL",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**NLTK Package**"
      ]
    },
    {
      "metadata": {
        "id": "EZHqlTRWnAQG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import nltk"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GmuxZyS3nM0D",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "tokens = nltk.word_tokenize(text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "K8tvUIK_nT6N",
        "colab_type": "code",
        "outputId": "d20b0125-a305-4350-b146-cd8e307f2e83",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        }
      },
      "cell_type": "code",
      "source": [
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "metadata": {
        "id": "joA2c2QCnbqA",
        "colab_type": "code",
        "outputId": "d380185d-f941-443b-e859-1e60a2ce007f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "tokens[:5]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Mr.', 'and', 'Mrs.', 'Dursley', ',']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "metadata": {
        "id": "0MnQ7xDYngWV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from nltk import tokenize"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6ESR6dg0n35e",
        "colab_type": "code",
        "outputId": "120eca85-a607-488c-fa14-3e1b1f155881",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "cell_type": "code",
      "source": [
        "sent_tokens = tokenize.sent_tokenize(text)\n",
        "nltk.sent_tokenize(text)\n",
        "sent_tokens"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Mr. and Mrs. Dursley, of number four, Privet Drive, were proud to say that they were perfectly normal, thank you very much.',\n",
              " 'They were the last people you’d expect to be involved in anything strange or mysterious, because they just didn’t hold with such nonsense.']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "metadata": {
        "id": "5Ry7dDxioLJc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from nltk.tokenize import MWETokenizer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tkdimfH86a-u",
        "colab_type": "code",
        "outputId": "449782ba-fcc8-454e-cef8-1a6e77ad3dce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "cell_type": "code",
      "source": [
        "tokenizer = MWETokenizer()\n",
        "tokenizer.add_mwe(('in', 'spite', 'of'))\n",
        "tokenizer.tokenize('In a little or a little bit or a lot in spite of'.split())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['In',\n",
              " 'a',\n",
              " 'little',\n",
              " 'or',\n",
              " 'a',\n",
              " 'little',\n",
              " 'bit',\n",
              " 'or',\n",
              " 'a',\n",
              " 'lot',\n",
              " 'in_spite_of']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "metadata": {
        "id": "x0aGmEXW_uf9",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**PoS tags for English**\n",
        "https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html"
      ]
    },
    {
      "metadata": {
        "id": "nyDRZa-k-gm7",
        "colab_type": "code",
        "outputId": "1ab0aa34-66cc-4dc6-b742-823180a5e506",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "cell_type": "code",
      "source": [
        "# nltk.pos_tag(tokens[10:20])\n",
        "nltk.pos_tag(tokens[9:20])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('Privet', 'NNP'),\n",
              " ('Drive', 'NNP'),\n",
              " (',', ','),\n",
              " ('were', 'VBD'),\n",
              " ('proud', 'JJ'),\n",
              " ('to', 'TO'),\n",
              " ('say', 'VB'),\n",
              " ('that', 'IN'),\n",
              " ('they', 'PRP'),\n",
              " ('were', 'VBD'),\n",
              " ('perfectly', 'RB')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "metadata": {
        "id": "rqZqB9rv6dFB",
        "colab_type": "code",
        "outputId": "878130c9-8c2a-42ce-a26f-3232e5754b38",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "cell_type": "code",
      "source": [
        "from nltk.corpus import wordnet\n",
        "nltk.download('wordnet')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "metadata": {
        "id": "md8yQC0vA9vp",
        "colab_type": "code",
        "outputId": "37285a14-33fa-412c-d011-45ea7f029713",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 148
        }
      },
      "cell_type": "code",
      "source": [
        "syn = wordnet.synsets(\"building\")\n",
        "\n",
        "print(syn)\n",
        "print(syn[0].name())\n",
        "print(syn[0].definition())\n",
        "print(syn[0].examples())\n",
        " \n",
        "print(syn[1].name())\n",
        "print(syn[1].definition())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[Synset('building.n.01'), Synset('construction.n.01'), Synset('construction.n.07'), Synset('building.n.04'), Synset('construct.v.01'), Synset('build_up.v.02'), Synset('build.v.03'), Synset('build.v.04'), Synset('build.v.05'), Synset('build.v.06'), Synset('build.v.07'), Synset('build.v.08'), Synset('build_up.v.04'), Synset('build.v.10')]\n",
            "building.n.01\n",
            "a structure that has a roof and walls and stands more or less permanently in one place\n",
            "['there was a three-story building on the corner', 'it was an imposing edifice']\n",
            "construction.n.01\n",
            "the act of constructing something\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "9n1mi-7NBD31",
        "colab_type": "code",
        "outputId": "23979867-ee33-45c9-facb-654f62ea0cd9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "cell_type": "code",
      "source": [
        "from nltk.corpus import stopwords\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "metadata": {
        "id": "SxgE4EHtByhU",
        "colab_type": "code",
        "outputId": "702ee502-376f-47cc-b8cf-01be9684bc60",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "\tstopwords.words('english')[:10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\"]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "metadata": {
        "id": "iM4MfFUTCAck",
        "colab_type": "code",
        "outputId": "76386db2-8ec7-4fd0-a504-a3b6b59e60d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        }
      },
      "cell_type": "code",
      "source": [
        "freq_dist = nltk.FreqDist(tokens)\n",
        "freq_dist.most_common(10)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(',', 5),\n",
              " ('were', 3),\n",
              " ('to', 2),\n",
              " ('they', 2),\n",
              " ('you', 2),\n",
              " ('.', 2),\n",
              " ('’', 2),\n",
              " ('Mr.', 1),\n",
              " ('and', 1),\n",
              " ('Mrs.', 1)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "metadata": {
        "id": "xs8do5aPCCdK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from nltk.stem import WordNetLemmatizer\n",
        "wordnet_lemmatizer = WordNetLemmatizer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7O-9CQvwEU_-",
        "colab_type": "code",
        "outputId": "78cc6539-119f-4005-c0b7-6047dbf88f39",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 500
        }
      },
      "cell_type": "code",
      "source": [
        "wiki_text = 'A lemma is a word that stands at the head of a definition in a dictionary. All the head words in a dictionary are lemmas'\n",
        "wiki_tokens = nltk.word_tokenize(wiki_text)\n",
        "for word in wiki_tokens:\n",
        "    print (\"{0:15}{1:15}\".format(word, wordnet_lemmatizer.lemmatize(word)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "A              A              \n",
            "lemma          lemma          \n",
            "is             is             \n",
            "a              a              \n",
            "word           word           \n",
            "that           that           \n",
            "stands         stand          \n",
            "at             at             \n",
            "the            the            \n",
            "head           head           \n",
            "of             of             \n",
            "a              a              \n",
            "definition     definition     \n",
            "in             in             \n",
            "a              a              \n",
            "dictionary     dictionary     \n",
            ".              .              \n",
            "All            All            \n",
            "the            the            \n",
            "head           head           \n",
            "words          word           \n",
            "in             in             \n",
            "a              a              \n",
            "dictionary     dictionary     \n",
            "are            are            \n",
            "lemmas         lemma          \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "beKM7NrPJCQh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from nltk.stem.snowball import SnowballStemmer\n",
        "\n",
        "stemmer = SnowballStemmer('english')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8LkQlDhsKdHb",
        "colab_type": "code",
        "outputId": "f1d878de-8f6d-406e-c56b-fef7d02a940a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 500
        }
      },
      "cell_type": "code",
      "source": [
        "for word in wiki_tokens:\n",
        "    print (\"{0:15}{1:15}\".format(word, stemmer.stem(word)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "A              a              \n",
            "lemma          lemma          \n",
            "is             is             \n",
            "a              a              \n",
            "word           word           \n",
            "that           that           \n",
            "stands         stand          \n",
            "at             at             \n",
            "the            the            \n",
            "head           head           \n",
            "of             of             \n",
            "a              a              \n",
            "definition     definit        \n",
            "in             in             \n",
            "a              a              \n",
            "dictionary     dictionari     \n",
            ".              .              \n",
            "All            all            \n",
            "the            the            \n",
            "head           head           \n",
            "words          word           \n",
            "in             in             \n",
            "a              a              \n",
            "dictionary     dictionari     \n",
            "are            are            \n",
            "lemmas         lemma          \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "WO4rrXBkLXMq",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Data Vectorizing**"
      ]
    },
    {
      "metadata": {
        "id": "INAVq4DRQKiD",
        "colab_type": "code",
        "outputId": "2920c817-4398-429d-d1cd-9437507994e7",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": "OK"
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-65c066ed-3929-42ed-8b5d-9319899cbbee\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-65c066ed-3929-42ed-8b5d-9319899cbbee\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving imdb_sampled.csv to imdb_sampled.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "hHyAN9ItQcH8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('imdb_sampled.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Y7pEmtA1b3Zt",
        "colab_type": "code",
        "outputId": "e89c4625-c354-4650-cb8e-187dd6e4e7dc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>type</th>\n",
              "      <th>review</th>\n",
              "      <th>label</th>\n",
              "      <th>file</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>48366</td>\n",
              "      <td>train</td>\n",
              "      <td>Not sure why it doesn't play in Peoria, appare...</td>\n",
              "      <td>pos</td>\n",
              "      <td>8530_9.txt</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>43357</td>\n",
              "      <td>train</td>\n",
              "      <td>A complex story laid on the background of part...</td>\n",
              "      <td>pos</td>\n",
              "      <td>4021_7.txt</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>40357</td>\n",
              "      <td>train</td>\n",
              "      <td>Beat a path to this important documentary that...</td>\n",
              "      <td>pos</td>\n",
              "      <td>1321_10.txt</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>37230</td>\n",
              "      <td>train</td>\n",
              "      <td>It's rare that I feel a need to write a review...</td>\n",
              "      <td>neg</td>\n",
              "      <td>9759_2.txt</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>27703</td>\n",
              "      <td>train</td>\n",
              "      <td>I like to keep my reviews short and simple, bu...</td>\n",
              "      <td>neg</td>\n",
              "      <td>12433_3.txt</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0   type                                             review label  \\\n",
              "0       48366  train  Not sure why it doesn't play in Peoria, appare...   pos   \n",
              "1       43357  train  A complex story laid on the background of part...   pos   \n",
              "2       40357  train  Beat a path to this important documentary that...   pos   \n",
              "3       37230  train  It's rare that I feel a need to write a review...   neg   \n",
              "4       27703  train  I like to keep my reviews short and simple, bu...   neg   \n",
              "\n",
              "          file  \n",
              "0   8530_9.txt  \n",
              "1   4021_7.txt  \n",
              "2  1321_10.txt  \n",
              "3   9759_2.txt  \n",
              "4  12433_3.txt  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "metadata": {
        "id": "IVyFZ4e6Xfbu",
        "colab_type": "code",
        "outputId": "3c50eda5-f094-4adc-cd9e-9913860fe734",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 56
        }
      },
      "cell_type": "code",
      "source": [
        "df['review'][0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Not sure why it doesn\\'t play in Peoria, apparently, but this is a very funny, clever British comedy. It\\'s set at the end of the \"swinging sixties\". Peter Sellars is fantastic as the rich, forty-something serial womaniser. The perfectly delectable Goldie Hawn, playing a 19 year American girl in London, is, initially, Sellars\\' \"catch of the day\". But the urbane TV food critic can\\'t stop himself from falling for the dizzy American blond.<br /><br />Humour, pathos, great script, strong performances from the leads and supporting caste.<br /><br />It\\'s a great film, and the best gag is the very last line.<br /><br />Try it, you\\'ll like it.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "metadata": {
        "id": "cabBBgmFb4ZP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "corpus = \" \".join([x for x in df['review']])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yAey_xcZc2h0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "corpus_tokens = nltk.word_tokenize(corpus)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SloYG3_NdRoe",
        "colab_type": "code",
        "outputId": "c6652ea1-e515-4cc5-d599-d459805c7c1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        }
      },
      "cell_type": "code",
      "source": [
        "freq_dist = nltk.FreqDist(corpus_tokens)\n",
        "freq_dist.most_common(15)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('the', 56925),\n",
              " (',', 54445),\n",
              " ('.', 46610),\n",
              " ('a', 30925),\n",
              " ('and', 30682),\n",
              " ('of', 28584),\n",
              " ('to', 26245),\n",
              " ('is', 21456),\n",
              " ('/', 20084),\n",
              " ('>', 20082),\n",
              " ('<', 20063),\n",
              " ('br', 20040),\n",
              " ('in', 17267),\n",
              " ('I', 16314),\n",
              " ('it', 15152)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "metadata": {
        "id": "JYomBtkjdVfy",
        "colab_type": "code",
        "outputId": "b6a3faae-c086-44d6-a70a-2e665c70bdf8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "from bs4 import BeautifulSoup\n",
        "cleantext = BeautifulSoup(\"<html>What a beautiful website!</html>\", \"lxml\").text\n",
        "cleantext"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'What a beautiful website!'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "metadata": {
        "id": "3i0FBNuodxDl",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df['review_clean'] = df['review'].apply(lambda x: BeautifulSoup(x.lower(), \"lxml\").text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_24jM9M6eJID",
        "colab_type": "code",
        "outputId": "0f08fcfa-864c-4d0d-b67f-81198fa7f6c8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        }
      },
      "cell_type": "code",
      "source": [
        "corpus_cl = \" \".join([x for x in df['review_clean']])\n",
        "corpus_cl_tokens = nltk.word_tokenize(corpus_cl)\n",
        "freq_dist = nltk.FreqDist(corpus_cl_tokens)\n",
        "freq_dist.most_common(15)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('the', 64760),\n",
              " (',', 54443),\n",
              " ('.', 46013),\n",
              " ('a', 32062),\n",
              " ('and', 31973),\n",
              " ('of', 28901),\n",
              " ('to', 26557),\n",
              " ('is', 21677),\n",
              " ('it', 18513),\n",
              " ('in', 18287),\n",
              " ('i', 16613),\n",
              " ('this', 14767),\n",
              " ('that', 14107),\n",
              " (\"'s\", 12014),\n",
              " ('was', 9841)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "metadata": {
        "id": "bjxa_giZe20_",
        "colab_type": "code",
        "outputId": "3ee6dc4a-5ec4-49f7-842a-318eb899d203",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>type</th>\n",
              "      <th>review</th>\n",
              "      <th>label</th>\n",
              "      <th>file</th>\n",
              "      <th>review_clean</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>48366</td>\n",
              "      <td>train</td>\n",
              "      <td>Not sure why it doesn't play in Peoria, appare...</td>\n",
              "      <td>pos</td>\n",
              "      <td>8530_9.txt</td>\n",
              "      <td>not sure why it doesn't play in peoria, appare...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>43357</td>\n",
              "      <td>train</td>\n",
              "      <td>A complex story laid on the background of part...</td>\n",
              "      <td>pos</td>\n",
              "      <td>4021_7.txt</td>\n",
              "      <td>a complex story laid on the background of part...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>40357</td>\n",
              "      <td>train</td>\n",
              "      <td>Beat a path to this important documentary that...</td>\n",
              "      <td>pos</td>\n",
              "      <td>1321_10.txt</td>\n",
              "      <td>beat a path to this important documentary that...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>37230</td>\n",
              "      <td>train</td>\n",
              "      <td>It's rare that I feel a need to write a review...</td>\n",
              "      <td>neg</td>\n",
              "      <td>9759_2.txt</td>\n",
              "      <td>it's rare that i feel a need to write a review...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>27703</td>\n",
              "      <td>train</td>\n",
              "      <td>I like to keep my reviews short and simple, bu...</td>\n",
              "      <td>neg</td>\n",
              "      <td>12433_3.txt</td>\n",
              "      <td>i like to keep my reviews short and simple, bu...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0   type                                             review label  \\\n",
              "0       48366  train  Not sure why it doesn't play in Peoria, appare...   pos   \n",
              "1       43357  train  A complex story laid on the background of part...   pos   \n",
              "2       40357  train  Beat a path to this important documentary that...   pos   \n",
              "3       37230  train  It's rare that I feel a need to write a review...   neg   \n",
              "4       27703  train  I like to keep my reviews short and simple, bu...   neg   \n",
              "\n",
              "          file                                       review_clean  \n",
              "0   8530_9.txt  not sure why it doesn't play in peoria, appare...  \n",
              "1   4021_7.txt  a complex story laid on the background of part...  \n",
              "2  1321_10.txt  beat a path to this important documentary that...  \n",
              "3   9759_2.txt  it's rare that i feel a need to write a review...  \n",
              "4  12433_3.txt  i like to keep my reviews short and simple, bu...  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "metadata": {
        "id": "1gd7Be0EfXTb",
        "colab_type": "code",
        "outputId": "3c7743a3-0b0a-4c49-b216-d679683c60e9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "cell_type": "code",
      "source": [
        "df['type'].value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "train    2500\n",
              "test     2500\n",
              "Name: type, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "metadata": {
        "id": "bic33UEae6Hh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train = df.loc[df['type'] == 'train']\n",
        "test = df.loc[df['type'] == 'test']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ydzc8pWfeXQN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "nnpUDfQGevn7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "count_vec = CountVectorizer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "KKqQ7ngZfrs9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "count_vec = count_vec.fit(train['review_clean'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "quDHGsAMfwiq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "transformed_train = count_vec.transform(train['review_clean'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SSvySKvNau9L",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "transformed_train = count_vec.fit_transform(train['review_clean'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jwkpTpxvgCg-",
        "colab_type": "code",
        "outputId": "b322023a-39a5-4377-90db-2352723817ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "type(transformed_train)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "scipy.sparse.csr.csr_matrix"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "metadata": {
        "id": "ORyFg2_2gXCx",
        "colab_type": "code",
        "outputId": "201feab5-b090-4102-934c-7394da25c073",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "transformed_train.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2500, 28528)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "metadata": {
        "id": "g-k_DSFAgcv5",
        "colab_type": "code",
        "outputId": "8ac6104d-b64b-40d7-aa38-29b67dbb6327",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        }
      },
      "cell_type": "code",
      "source": [
        "count_vec.get_feature_names()[-15:]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['zooming',\n",
              " 'zooms',\n",
              " 'zoot',\n",
              " 'zorak',\n",
              " 'zorie',\n",
              " 'zorrilla',\n",
              " 'zu',\n",
              " 'zulu',\n",
              " 'zulus',\n",
              " 'zuniga',\n",
              " 'zwartboek',\n",
              " 'zwick',\n",
              " 'zx81',\n",
              " 'zã',\n",
              " 'ã¼ber']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "metadata": {
        "id": "wAh20MlvghRD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "count_vec = CountVectorizer(lowercase=True, stop_words='english', strip_accents='unicode', min_df = 50, max_df=1000, max_features=500)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lGhYItXlhIB_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "count_vec = count_vec.fit(train['review_clean'])\n",
        "transformed_train = count_vec.transform(train['review_clean'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "56qApoXxhKGF",
        "colab_type": "code",
        "outputId": "83ffba03-b6d8-4c7f-9111-394fa55c36aa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "transformed_train.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2500, 500)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "metadata": {
        "id": "-pOTvscWhPTC",
        "colab_type": "code",
        "outputId": "ae368a2a-ba80-4a21-b96f-93000bde1368",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        }
      },
      "cell_type": "code",
      "source": [
        "count_vec.get_feature_names()[:10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['absolutely',\n",
              " 'act',\n",
              " 'actor',\n",
              " 'age',\n",
              " 'american',\n",
              " 'attempt',\n",
              " 'audience',\n",
              " 'awful',\n",
              " 'based',\n",
              " 'beautiful']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 67
        }
      ]
    },
    {
      "metadata": {
        "id": "nf9EqcjHhfCM",
        "colab_type": "code",
        "outputId": "2c4e822c-6e42-4512-a80d-f22ef16ab70d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "cell_type": "code",
      "source": [
        "transformed_train.toarray()[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "metadata": {
        "id": "GwIfBMjohs5V",
        "colab_type": "code",
        "outputId": "cc87ffba-93b0-44f1-a875-70a27cbb1206",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "'american' in train['review_clean'][0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "metadata": {
        "id": "wtce8_4IiB0c",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "tfidf_vec = TfidfVectorizer(max_features = 300)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "JEg26VwPj0LN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "tfidf_vec = tfidf_vec.fit(train['review_clean'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QbbuPV1Zj7vO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_tfidf = tfidf_vec.transform(train['review_clean'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fxpduOKNkEf9",
        "colab_type": "code",
        "outputId": "1d4ed6e6-69e4-4b1f-e648-24069386ebcb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1131
        }
      },
      "cell_type": "code",
      "source": [
        "train_tfidf.toarray()[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.1008419 , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.07148209, 0.0824425 , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.12980755, 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.13306956,\n",
              "       0.        , 0.09657877, 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.16872522, 0.        , 0.        , 0.        ,\n",
              "       0.1673935 , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.14412194, 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.13621194,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.07828459,\n",
              "       0.        , 0.        , 0.        , 0.06611639, 0.        ,\n",
              "       0.17262082, 0.        , 0.14794585, 0.        , 0.        ,\n",
              "       0.17598704, 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.2386249 , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.11072142,\n",
              "       0.        , 0.        , 0.        , 0.21755592, 0.        ,\n",
              "       0.27758547, 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.16200722, 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.08702831, 0.        , 0.16462646, 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.07521481, 0.        , 0.        , 0.10320975, 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.1576904 , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.17394305, 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.1364444 ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.16240119, 0.        , 0.        , 0.        , 0.49422774,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.05397525,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.17198084, 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.2050094 , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.14147001,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.17079316,\n",
              "       0.        , 0.        , 0.07857018, 0.        , 0.        ])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 108
        }
      ]
    },
    {
      "metadata": {
        "id": "v-XGi8bFd2WL",
        "colab_type": "code",
        "outputId": "a3530046-220a-45ea-83fc-4e5da1a4e38f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 5587
        }
      },
      "cell_type": "code",
      "source": [
        "tfidf_vec.get_feature_names()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['10',\n",
              " 'about',\n",
              " 'acting',\n",
              " 'action',\n",
              " 'actors',\n",
              " 'actually',\n",
              " 'after',\n",
              " 'again',\n",
              " 'all',\n",
              " 'almost',\n",
              " 'also',\n",
              " 'although',\n",
              " 'always',\n",
              " 'am',\n",
              " 'an',\n",
              " 'and',\n",
              " 'another',\n",
              " 'any',\n",
              " 'anyone',\n",
              " 'anything',\n",
              " 'are',\n",
              " 'around',\n",
              " 'as',\n",
              " 'at',\n",
              " 'audience',\n",
              " 'away',\n",
              " 'back',\n",
              " 'bad',\n",
              " 'be',\n",
              " 'beautiful',\n",
              " 'because',\n",
              " 'been',\n",
              " 'before',\n",
              " 'being',\n",
              " 'believe',\n",
              " 'best',\n",
              " 'better',\n",
              " 'between',\n",
              " 'big',\n",
              " 'bit',\n",
              " 'black',\n",
              " 'book',\n",
              " 'both',\n",
              " 'budget',\n",
              " 'but',\n",
              " 'by',\n",
              " 'can',\n",
              " 'cast',\n",
              " 'character',\n",
              " 'characters',\n",
              " 'come',\n",
              " 'comedy',\n",
              " 'comes',\n",
              " 'could',\n",
              " 'course',\n",
              " 'day',\n",
              " 'did',\n",
              " 'didn',\n",
              " 'different',\n",
              " 'director',\n",
              " 'do',\n",
              " 'does',\n",
              " 'doesn',\n",
              " 'don',\n",
              " 'done',\n",
              " 'down',\n",
              " 'dvd',\n",
              " 'each',\n",
              " 'effects',\n",
              " 'end',\n",
              " 'ending',\n",
              " 'enough',\n",
              " 'episode',\n",
              " 'especially',\n",
              " 'even',\n",
              " 'ever',\n",
              " 'every',\n",
              " 'everyone',\n",
              " 'everything',\n",
              " 'fact',\n",
              " 'family',\n",
              " 'far',\n",
              " 'feel',\n",
              " 'few',\n",
              " 'film',\n",
              " 'films',\n",
              " 'find',\n",
              " 'first',\n",
              " 'for',\n",
              " 'found',\n",
              " 'from',\n",
              " 'fun',\n",
              " 'funny',\n",
              " 'get',\n",
              " 'gets',\n",
              " 'girl',\n",
              " 'give',\n",
              " 'go',\n",
              " 'goes',\n",
              " 'going',\n",
              " 'good',\n",
              " 'got',\n",
              " 'great',\n",
              " 'guy',\n",
              " 'had',\n",
              " 'half',\n",
              " 'hard',\n",
              " 'has',\n",
              " 'have',\n",
              " 'having',\n",
              " 'he',\n",
              " 'her',\n",
              " 'here',\n",
              " 'him',\n",
              " 'his',\n",
              " 'horror',\n",
              " 'how',\n",
              " 'however',\n",
              " 'if',\n",
              " 'in',\n",
              " 'instead',\n",
              " 'interesting',\n",
              " 'into',\n",
              " 'is',\n",
              " 'isn',\n",
              " 'it',\n",
              " 'its',\n",
              " 'john',\n",
              " 'just',\n",
              " 'kind',\n",
              " 'know',\n",
              " 'last',\n",
              " 'least',\n",
              " 'left',\n",
              " 'let',\n",
              " 'life',\n",
              " 'like',\n",
              " 'little',\n",
              " 'll',\n",
              " 'long',\n",
              " 'look',\n",
              " 'looking',\n",
              " 'looks',\n",
              " 'lot',\n",
              " 'love',\n",
              " 'made',\n",
              " 'make',\n",
              " 'makes',\n",
              " 'making',\n",
              " 'man',\n",
              " 'many',\n",
              " 'may',\n",
              " 'maybe',\n",
              " 'me',\n",
              " 'might',\n",
              " 'minutes',\n",
              " 'money',\n",
              " 'more',\n",
              " 'most',\n",
              " 'movie',\n",
              " 'movies',\n",
              " 'much',\n",
              " 'music',\n",
              " 'must',\n",
              " 'my',\n",
              " 'never',\n",
              " 'new',\n",
              " 'nice',\n",
              " 'night',\n",
              " 'no',\n",
              " 'not',\n",
              " 'nothing',\n",
              " 'now',\n",
              " 'of',\n",
              " 'off',\n",
              " 'old',\n",
              " 'on',\n",
              " 'once',\n",
              " 'one',\n",
              " 'only',\n",
              " 'or',\n",
              " 'original',\n",
              " 'other',\n",
              " 'our',\n",
              " 'out',\n",
              " 'over',\n",
              " 'own',\n",
              " 'part',\n",
              " 'people',\n",
              " 'performance',\n",
              " 'place',\n",
              " 'played',\n",
              " 'plot',\n",
              " 'point',\n",
              " 'poor',\n",
              " 'pretty',\n",
              " 'probably',\n",
              " 'put',\n",
              " 'quite',\n",
              " 'rather',\n",
              " 're',\n",
              " 'real',\n",
              " 'really',\n",
              " 'reason',\n",
              " 'right',\n",
              " 'role',\n",
              " 'said',\n",
              " 'same',\n",
              " 'saw',\n",
              " 'say',\n",
              " 'scene',\n",
              " 'scenes',\n",
              " 'screen',\n",
              " 'script',\n",
              " 'see',\n",
              " 'seem',\n",
              " 'seems',\n",
              " 'seen',\n",
              " 'sense',\n",
              " 'series',\n",
              " 'set',\n",
              " 'she',\n",
              " 'should',\n",
              " 'show',\n",
              " 'shows',\n",
              " 'since',\n",
              " 'so',\n",
              " 'some',\n",
              " 'someone',\n",
              " 'something',\n",
              " 'special',\n",
              " 'star',\n",
              " 'still',\n",
              " 'story',\n",
              " 'such',\n",
              " 'sure',\n",
              " 'take',\n",
              " 'than',\n",
              " 'that',\n",
              " 'the',\n",
              " 'their',\n",
              " 'them',\n",
              " 'then',\n",
              " 'there',\n",
              " 'these',\n",
              " 'they',\n",
              " 'thing',\n",
              " 'things',\n",
              " 'think',\n",
              " 'this',\n",
              " 'those',\n",
              " 'though',\n",
              " 'thought',\n",
              " 'through',\n",
              " 'time',\n",
              " 'times',\n",
              " 'to',\n",
              " 'together',\n",
              " 'too',\n",
              " 'true',\n",
              " 'trying',\n",
              " 'tv',\n",
              " 'two',\n",
              " 'up',\n",
              " 'us',\n",
              " 've',\n",
              " 'very',\n",
              " 'want',\n",
              " 'was',\n",
              " 'wasn',\n",
              " 'watch',\n",
              " 'watched',\n",
              " 'watching',\n",
              " 'way',\n",
              " 'we',\n",
              " 'well',\n",
              " 'were',\n",
              " 'what',\n",
              " 'when',\n",
              " 'where',\n",
              " 'which',\n",
              " 'while',\n",
              " 'who',\n",
              " 'whole',\n",
              " 'why',\n",
              " 'wife',\n",
              " 'will',\n",
              " 'with',\n",
              " 'without',\n",
              " 'woman',\n",
              " 'work',\n",
              " 'world',\n",
              " 'worst',\n",
              " 'would',\n",
              " 'year',\n",
              " 'years',\n",
              " 'yet',\n",
              " 'you',\n",
              " 'young',\n",
              " 'your']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 109
        }
      ]
    },
    {
      "metadata": {
        "id": "fHHQSmWgkHBI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_tfidf_array = train_tfidf.toarray()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BMHJP8TseApB",
        "colab_type": "code",
        "outputId": "1200bb4a-cbbe-4ca0-89e6-a56ce957ae99",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "type(train_tfidf_array)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "numpy.ndarray"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 111
        }
      ]
    },
    {
      "metadata": {
        "id": "A_e4YQ7NkRlr",
        "colab_type": "code",
        "outputId": "fa659797-3fd0-4f95-f403-404d81d15ca8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "np.max(train_tfidf_array)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7930853274278427"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 112
        }
      ]
    },
    {
      "metadata": {
        "id": "gBrHC1pCkYby",
        "colab_type": "code",
        "outputId": "8193bb79-a129-48a3-f3a6-aa415f17c9c1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>type</th>\n",
              "      <th>review</th>\n",
              "      <th>label</th>\n",
              "      <th>file</th>\n",
              "      <th>review_clean</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>48366</td>\n",
              "      <td>train</td>\n",
              "      <td>Not sure why it doesn't play in Peoria, appare...</td>\n",
              "      <td>pos</td>\n",
              "      <td>8530_9.txt</td>\n",
              "      <td>not sure why it doesn't play in peoria, appare...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>43357</td>\n",
              "      <td>train</td>\n",
              "      <td>A complex story laid on the background of part...</td>\n",
              "      <td>pos</td>\n",
              "      <td>4021_7.txt</td>\n",
              "      <td>a complex story laid on the background of part...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>40357</td>\n",
              "      <td>train</td>\n",
              "      <td>Beat a path to this important documentary that...</td>\n",
              "      <td>pos</td>\n",
              "      <td>1321_10.txt</td>\n",
              "      <td>beat a path to this important documentary that...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>37230</td>\n",
              "      <td>train</td>\n",
              "      <td>It's rare that I feel a need to write a review...</td>\n",
              "      <td>neg</td>\n",
              "      <td>9759_2.txt</td>\n",
              "      <td>it's rare that i feel a need to write a review...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>27703</td>\n",
              "      <td>train</td>\n",
              "      <td>I like to keep my reviews short and simple, bu...</td>\n",
              "      <td>neg</td>\n",
              "      <td>12433_3.txt</td>\n",
              "      <td>i like to keep my reviews short and simple, bu...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0   type                                             review label  \\\n",
              "0       48366  train  Not sure why it doesn't play in Peoria, appare...   pos   \n",
              "1       43357  train  A complex story laid on the background of part...   pos   \n",
              "2       40357  train  Beat a path to this important documentary that...   pos   \n",
              "3       37230  train  It's rare that I feel a need to write a review...   neg   \n",
              "4       27703  train  I like to keep my reviews short and simple, bu...   neg   \n",
              "\n",
              "          file                                       review_clean  \n",
              "0   8530_9.txt  not sure why it doesn't play in peoria, appare...  \n",
              "1   4021_7.txt  a complex story laid on the background of part...  \n",
              "2  1321_10.txt  beat a path to this important documentary that...  \n",
              "3   9759_2.txt  it's rare that i feel a need to write a review...  \n",
              "4  12433_3.txt  i like to keep my reviews short and simple, bu...  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 113
        }
      ]
    },
    {
      "metadata": {
        "id": "Mxgdpmi0kiCr",
        "colab_type": "code",
        "outputId": "567f90d7-eccd-42c5-b607-c14e863aed45",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "cell_type": "code",
      "source": [
        "df['label'].value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pos    2511\n",
              "neg    2489\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 114
        }
      ]
    },
    {
      "metadata": {
        "id": "nc7CCogxlOmk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import LabelEncoder"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UXmiCXwHlRhX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "label_enc = LabelEncoder()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PMd4DdDZlpUs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "label_enc = label_enc.fit(df['label'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "b4loF70tluTz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "Y_train = label_enc.transform(train['label'])\n",
        "Y_test = label_enc.transform(test['label'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "qYNafGN7l6Cz",
        "colab_type": "code",
        "outputId": "dee4b214-d12e-427d-8b16-50464c479c9b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "Y_train[:5]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 1, 1, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 119
        }
      ]
    },
    {
      "metadata": {
        "id": "24yIXRBpl1rk",
        "colab_type": "code",
        "outputId": "9b46b27b-04f0-47bc-b211-76796adb68e4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "cell_type": "code",
      "source": [
        "train.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>type</th>\n",
              "      <th>review</th>\n",
              "      <th>label</th>\n",
              "      <th>file</th>\n",
              "      <th>review_clean</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>48366</td>\n",
              "      <td>train</td>\n",
              "      <td>Not sure why it doesn't play in Peoria, appare...</td>\n",
              "      <td>pos</td>\n",
              "      <td>8530_9.txt</td>\n",
              "      <td>not sure why it doesn't play in peoria, appare...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>43357</td>\n",
              "      <td>train</td>\n",
              "      <td>A complex story laid on the background of part...</td>\n",
              "      <td>pos</td>\n",
              "      <td>4021_7.txt</td>\n",
              "      <td>a complex story laid on the background of part...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>40357</td>\n",
              "      <td>train</td>\n",
              "      <td>Beat a path to this important documentary that...</td>\n",
              "      <td>pos</td>\n",
              "      <td>1321_10.txt</td>\n",
              "      <td>beat a path to this important documentary that...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>37230</td>\n",
              "      <td>train</td>\n",
              "      <td>It's rare that I feel a need to write a review...</td>\n",
              "      <td>neg</td>\n",
              "      <td>9759_2.txt</td>\n",
              "      <td>it's rare that i feel a need to write a review...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>27703</td>\n",
              "      <td>train</td>\n",
              "      <td>I like to keep my reviews short and simple, bu...</td>\n",
              "      <td>neg</td>\n",
              "      <td>12433_3.txt</td>\n",
              "      <td>i like to keep my reviews short and simple, bu...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0   type                                             review label  \\\n",
              "0       48366  train  Not sure why it doesn't play in Peoria, appare...   pos   \n",
              "1       43357  train  A complex story laid on the background of part...   pos   \n",
              "2       40357  train  Beat a path to this important documentary that...   pos   \n",
              "3       37230  train  It's rare that I feel a need to write a review...   neg   \n",
              "4       27703  train  I like to keep my reviews short and simple, bu...   neg   \n",
              "\n",
              "          file                                       review_clean  \n",
              "0   8530_9.txt  not sure why it doesn't play in peoria, appare...  \n",
              "1   4021_7.txt  a complex story laid on the background of part...  \n",
              "2  1321_10.txt  beat a path to this important documentary that...  \n",
              "3   9759_2.txt  it's rare that i feel a need to write a review...  \n",
              "4  12433_3.txt  i like to keep my reviews short and simple, bu...  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 120
        }
      ]
    },
    {
      "metadata": {
        "id": "n0sNFA9ckmdU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.svm import LinearSVC"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "CBagTgq3l_x8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "X_train = train_tfidf_array\n",
        "X_test = tfidf_vec.transform(test['review_clean']).toarray()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bRWlJxehmM2P",
        "colab_type": "code",
        "outputId": "f7800a89-2a1f-47dd-b041-6886a57c4aa3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1131
        }
      },
      "cell_type": "code",
      "source": [
        "X_test[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.        , 0.13754444, 0.06258271, 0.        , 0.        ,\n",
              "       0.14383747, 0.        , 0.07367021, 0.04114035, 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.05046396, 0.        , 0.0616752 , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.17885765, 0.08251281, 0.        ,\n",
              "       0.        , 0.07118599, 0.        , 0.03892757, 0.        ,\n",
              "       0.        , 0.11516334, 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.03329576,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.13189255, 0.        , 0.        , 0.        ,\n",
              "       0.05803834, 0.06899018, 0.        , 0.05854284, 0.        ,\n",
              "       0.        , 0.        , 0.08717125, 0.        , 0.06816406,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.05249217,\n",
              "       0.06445511, 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.13234557, 0.08387796,\n",
              "       0.08638402, 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.04887982, 0.        , 0.11941421, 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.07743376, 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.22658145, 0.05540793,\n",
              "       0.08780883, 0.15984386, 0.05687555, 0.13608842, 0.        ,\n",
              "       0.25004001, 0.        , 0.        , 0.04641569, 0.        ,\n",
              "       0.06563024, 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.04355127, 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.08558714, 0.        , 0.05769504, 0.21858935,\n",
              "       0.        , 0.10960805, 0.        , 0.        , 0.10306134,\n",
              "       0.06460523, 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.03763948, 0.07388904, 0.07274387, 0.10329778, 0.        ,\n",
              "       0.        , 0.109726  , 0.        , 0.03885714, 0.05185695,\n",
              "       0.04603587, 0.        , 0.        , 0.        , 0.04622505,\n",
              "       0.        , 0.16374198, 0.        , 0.05925897, 0.        ,\n",
              "       0.        , 0.08630559, 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.07309603, 0.        , 0.05420533, 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.0746329 , 0.07709345, 0.        ,\n",
              "       0.        , 0.        , 0.08679651, 0.        , 0.05264587,\n",
              "       0.        , 0.        , 0.06323854, 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.06828039,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.0894369 , 0.3462545 ,\n",
              "       0.        , 0.06171979, 0.        , 0.        , 0.13870585,\n",
              "       0.04545393, 0.14506994, 0.        , 0.        , 0.16206385,\n",
              "       0.        , 0.07579273, 0.        , 0.07003221, 0.15371328,\n",
              "       0.        , 0.20942815, 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.06460523, 0.        , 0.        ,\n",
              "       0.14146166, 0.        , 0.07587204, 0.        , 0.        ,\n",
              "       0.        , 0.08819948, 0.07098988, 0.        , 0.        ,\n",
              "       0.05420533, 0.        , 0.0472924 , 0.05004435, 0.        ,\n",
              "       0.10756369, 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "       0.        , 0.        , 0.        , 0.10660423, 0.        ,\n",
              "       0.        , 0.        , 0.19659298, 0.        , 0.06781835])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 123
        }
      ]
    },
    {
      "metadata": {
        "id": "Tr9US_G8mOTk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = LinearSVC()\n",
        "model.fit(X_train, Y_train)\n",
        "y_pred = model.predict(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "n8L4qQnCf5G_",
        "colab_type": "code",
        "outputId": "3cd03000-30a4-44b3-c632-3905180f6c6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "len(Y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2500"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 125
        }
      ]
    },
    {
      "metadata": {
        "id": "4z2lxNd7magV",
        "colab_type": "code",
        "outputId": "d35d0607-df0e-4498-f873-a4e0a14b30f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "cell_type": "code",
      "source": [
        "acc = accuracy_score(Y_test, y_pred)\n",
        "\n",
        "print(\"Accuracy on the IMDB dataset: {:.2f}\".format(acc*100))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy on the IMDB dataset: 77.96\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "G6vd19nHmjZv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WHvKP8_0nWsj",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Word Embeddings**\n",
        "\n",
        "\n",
        "\n",
        "1.   Word2Vec\n",
        "  ![](https://fasttext.cc/img/cbo_vs_skipgram.png)\n",
        "2.   FastText\n",
        "3.   Any you can think of\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "xRLEdxm_m9HK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        ""
      ]
    },
    {
      "metadata": {
        "id": "uCccxtLTnHkD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}