count-tf-idf-notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "count_tf_idf.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8gs4LslFVYv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from pandas import DataFrame"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Km4-r8KsFhUV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_document_term_matrix(message_list, vectorizer):\n",
        "    doc_term_matrix = vectorizer.fit_transform(message_list)\n",
        "    return DataFrame(doc_term_matrix.toarray(), \n",
        "                     columns=vectorizer.get_feature_names())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jFTR9b78GDmU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "msg_1 = [\"My name is Bhavesh\",\n",
        "        \"Please subscribe to my YouTube channel\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VTUZYWjFGXN0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "count_vect = CountVectorizer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c8tprPLMGbp5",
        "colab_type": "code",
        "outputId": "78158e87-f03e-4241-8705-68b81c690bc7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        }
      },
      "source": [
        "create_document_term_matrix(msg_1, count_vect)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>bhavesh</th>\n",
              "      <th>channel</th>\n",
              "      <th>is</th>\n",
              "      <th>my</th>\n",
              "      <th>name</th>\n",
              "      <th>please</th>\n",
              "      <th>subscribe</th>\n",
              "      <th>to</th>\n",
              "      <th>youtube</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   bhavesh  channel  is  my  name  please  subscribe  to  youtube\n",
              "0        1        0   1   1     1       0          0   0        0\n",
              "1        0        1   0   1     0       1          1   1        1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i9FqaTO-JCD4",
        "colab_type": "text"
      },
      "source": [
        "# TF-IDF"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lyrGxv0-JEbl",
        "colab_type": "text"
      },
      "source": [
        "## Term Frequency (tf)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kZi1FgpoI6m8",
        "colab_type": "text"
      },
      "source": [
        "$$\n",
        "t f_{i, j}=\\frac{n_{i, j}}{\\sum_{k} n_{i, j}}\n",
        "$$"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9SgL3wCwJBXZ",
        "colab_type": "text"
      },
      "source": [
        "## Inverse Data Frequency (idf)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-ehyK7mHI9eX",
        "colab_type": "text"
      },
      "source": [
        "$$\n",
        "i d f(w)=\\log \\left(\\frac{N}{d f_{t}}\\right)\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vVUeG-xfGhBk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "msg_2 = [\"Bhavesh is my name\",\n",
        "         \"Bhavesh likes Python programming language\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zRGFGs00PzxF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tfidf_vect = TfidfVectorizer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KIP50r8dP4h5",
        "colab_type": "code",
        "outputId": "79e2fe12-8580-4c47-d415-070b26924b99",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        }
      },
      "source": [
        "create_document_term_matrix(msg_2, tfidf_vect)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>bhavesh</th>\n",
              "      <th>is</th>\n",
              "      <th>language</th>\n",
              "      <th>likes</th>\n",
              "      <th>my</th>\n",
              "      <th>name</th>\n",
              "      <th>programming</th>\n",
              "      <th>python</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.379978</td>\n",
              "      <td>0.534046</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.534046</td>\n",
              "      <td>0.534046</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.335176</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.471078</td>\n",
              "      <td>0.471078</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.471078</td>\n",
              "      <td>0.471078</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    bhavesh        is  language  ...      name  programming    python\n",
              "0  0.379978  0.534046  0.000000  ...  0.534046     0.000000  0.000000\n",
              "1  0.335176  0.000000  0.471078  ...  0.000000     0.471078  0.471078\n",
              "\n",
              "[2 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "74DpTD4nP8x8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "msg_3 = [\"Bhavesh Bhavesh Bhavesh is my name\",\n",
        "         \"Bhavesh likes Python programming language\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nVcFm_WkUDqR",
        "colab_type": "code",
        "outputId": "2dfbe2cb-007b-4eec-fa58-cb2a7fafda69",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        }
      },
      "source": [
        "create_document_term_matrix(msg_3, tfidf_vect)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>bhavesh</th>\n",
              "      <th>is</th>\n",
              "      <th>language</th>\n",
              "      <th>likes</th>\n",
              "      <th>my</th>\n",
              "      <th>name</th>\n",
              "      <th>programming</th>\n",
              "      <th>python</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.776515</td>\n",
              "      <td>0.363788</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.363788</td>\n",
              "      <td>0.363788</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.335176</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.471078</td>\n",
              "      <td>0.471078</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.471078</td>\n",
              "      <td>0.471078</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    bhavesh        is  language  ...      name  programming    python\n",
              "0  0.776515  0.363788  0.000000  ...  0.363788     0.000000  0.000000\n",
              "1  0.335176  0.000000  0.471078  ...  0.000000     0.471078  0.471078\n",
              "\n",
              "[2 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ge3NcWkAUJxZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "msg_4 = [\"Bhavesh Bhavesh Bhavesh is my name\",\n",
        "         \"I like Python programming language\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9gMHUqo3UZQx",
        "colab_type": "code",
        "outputId": "6a1e7221-ad92-4ffe-98fb-c378523ef158",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        }
      },
      "source": [
        "create_document_term_matrix(msg_4, tfidf_vect)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>bhavesh</th>\n",
              "      <th>is</th>\n",
              "      <th>language</th>\n",
              "      <th>like</th>\n",
              "      <th>my</th>\n",
              "      <th>name</th>\n",
              "      <th>programming</th>\n",
              "      <th>python</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.866025</td>\n",
              "      <td>0.288675</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.288675</td>\n",
              "      <td>0.288675</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    bhavesh        is  language  like        my      name  programming  python\n",
              "0  0.866025  0.288675       0.0   0.0  0.288675  0.288675          0.0     0.0\n",
              "1  0.000000  0.000000       0.5   0.5  0.000000  0.000000          0.5     0.5"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XdzcLOBlUba8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}