CountVectorizer_on_twitter_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CountVectorizer on twitter data.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMWQhh6Cc3yXcL4hxG+S2/M",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rohitgupta29/Actionable-NLP/blob/main/CountVectorizer_on_twitter_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ElltAdHgzzYd"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p--QqsgqzzwZ"
      },
      "source": [
        "Dataset - Twitter data from Kaggle. \n",
        "\n",
        "Steps - 1. Loaded and cleaned the data.\n",
        "        2. Used CountVectorizer on it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wRx_icFemmdc"
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GzFC1Adwzjsh"
      },
      "source": [
        "This Dataset - twitter Hatred data."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "koPZi_iKzvqn"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ahaihLoGzycX"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1RIOaCtTzyru"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I2EVSeY7ms-O"
      },
      "source": [
        "train = pd.read_csv(\"https://raw.githubusercontent.com/rohitgupta29/Datasets/master/NLP/train.csv\")"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "id": "zjSQv_jFmyW3",
        "outputId": "f9608b70-1d99-4c8f-f312-9e656cb38a7c"
      },
      "source": [
        "train.head()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>label</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>@user when a father is dysfunctional and is s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>@user @user thanks for #lyft credit i can't us...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>bihday your majesty</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>factsguide: society now    #motivation</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id  label                                              tweet\n",
              "0   1      0   @user when a father is dysfunctional and is s...\n",
              "1   2      0  @user @user thanks for #lyft credit i can't us...\n",
              "2   3      0                                bihday your majesty\n",
              "3   4      0  #model   i love u take with u all the time in ...\n",
              "4   5      0             factsguide: society now    #motivation"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ayOL-umjnMyf",
        "outputId": "fc137878-0131-461e-bab8-b2fc8cafd861"
      },
      "source": [
        "#non- racist\n",
        "sum(train[\"label\"] == 0)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "29720"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iXbLiexTnYUy",
        "outputId": "dfed4d8b-0d6e-4a47-d0b1-1f43abeb29b4"
      },
      "source": [
        "#racist\n",
        "sum(train[\"label\"] == 1)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2242"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QeQv5793nhaI"
      },
      "source": [
        "import re\n",
        "\n",
        "#set up punctuations we want to be replaced\n",
        "REPLACE_NO_SPACE = re.compile(\"(\\.)|(\\;)|(\\:)|(\\!)|(\\')|(\\?)|(\\,)|(\\\")|(\\|)|(\\()|(\\))|(\\[)|(\\])|(\\%)|(\\$)|(\\>)|(\\<)|(\\{)|(\\})\")\n",
        "REPLACE_WITH_SPACE = re.compile(\"(<br\\s/><br\\s/?)|(-)|(/)|(:).\")"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RxBbna4Fnv9w",
        "outputId": "10b2268d-cac6-44d4-a3de-2d82e1bcd6ab"
      },
      "source": [
        "!pip install tweet-preprocessor"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tweet-preprocessor\n",
            "  Downloading https://files.pythonhosted.org/packages/17/9d/71bd016a9edcef8860c607e531f30bd09b13103c7951ae73dd2bf174163c/tweet_preprocessor-0.6.0-py3-none-any.whl\n",
            "Installing collected packages: tweet-preprocessor\n",
            "Successfully installed tweet-preprocessor-0.6.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "71GAgxvpoArY"
      },
      "source": [
        "import preprocessor as p\n",
        "\n",
        "#its a custom function to clean the dataset \n",
        "def clean_tweets(df):\n",
        "  tempArr = []\n",
        "  for line in df:\n",
        "    # send to tweet_processor\n",
        "    tmpL = p.clean(line)\n",
        "    # remove puctuation\n",
        "    tmpL = REPLACE_NO_SPACE.sub(\"\", tmpL.lower()) # convert all tweets to lower cases\n",
        "    tmpL = REPLACE_WITH_SPACE.sub(\" \", tmpL)\n",
        "    tempArr.append(tmpL)\n",
        "  return tempArr"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iEHfgMrvpU1o"
      },
      "source": [
        "#cleaning training data\n",
        "train_tweet = clean_tweets(train[\"tweet\"])\n",
        "train_tweet = pd.DataFrame(train_tweet)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "id": "H7EUZxzFph0t",
        "outputId": "58661ee7-b4fa-4908-a5a8-77c451df3aa8"
      },
      "source": [
        "#append cleaned tweets to the training data\n",
        "train[\"clean_tweet\"] = train_tweet\n",
        "\n",
        "train1 = train.drop(['tweet','id','label'], axis = 1)\n",
        "\n",
        "train_vectorize = train1.iloc[:5]\n",
        "train_vectorize"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>clean_tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>when a father is dysfunctional and is so selfi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>thanks for credit i cant use cause they dont o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>bihday your majesty</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>i love u take with u all the time in ur</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>factsguide society now</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                         clean_tweet\n",
              "0  when a father is dysfunctional and is so selfi...\n",
              "1  thanks for credit i cant use cause they dont o...\n",
              "2                                bihday your majesty\n",
              "3            i love u take with u all the time in ur\n",
              "4                             factsguide society now"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y4PIgxTZqs24"
      },
      "source": [
        ""
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVMASckYq53C"
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U2CYLnhcq7_V",
        "outputId": "6a43a08b-71a0-4b23-d650-00cd2ed17c71"
      },
      "source": [
        "vectorizer = CountVectorizer()\n",
        "\n",
        "vectorizer = CountVectorizer(stop_words = 'english')\n",
        "\n",
        "vectorizer.fit(list(train_vectorize[\"clean_tweet\"]))  \n",
        "  "
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
              "                dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
              "                lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
              "                ngram_range=(1, 1), preprocessor=None, stop_words='english',\n",
              "                strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                tokenizer=None, vocabulary=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I8IZLap6rcF4",
        "outputId": "2fd42f2c-57a8-4adb-cc60-2b14f530d737"
      },
      "source": [
        "vectorizer.vocabulary_"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'bihday': 0,\n",
              " 'cause': 1,\n",
              " 'credit': 2,\n",
              " 'dont': 3,\n",
              " 'drags': 4,\n",
              " 'dysfunction': 5,\n",
              " 'dysfunctional': 6,\n",
              " 'factsguide': 7,\n",
              " 'father': 8,\n",
              " 'kids': 9,\n",
              " 'love': 10,\n",
              " 'majesty': 11,\n",
              " 'offer': 12,\n",
              " 'pdx': 13,\n",
              " 'selfish': 14,\n",
              " 'society': 15,\n",
              " 'thanks': 16,\n",
              " 'time': 17,\n",
              " 'ur': 18,\n",
              " 'use': 19,\n",
              " 'vans': 20,\n",
              " 'wheelchair': 21}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 339
        },
        "id": "nOA8uFw6wbga",
        "outputId": "b4a225b4-85a3-4ecd-fca3-8a4dee54ac85"
      },
      "source": [
        "vector = vectorizer.transform(list(train_vectorize['clean_tweet']))\n",
        "print(vector.shape)\n",
        "print(type(vector))\n",
        "print(vector.toarray())\n",
        "pd.DataFrame(vector.toarray(), columns = vectorizer.get_feature_names())"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(5, 22)\n",
            "<class 'scipy.sparse.csr.csr_matrix'>\n",
            "[[0 0 0 0 1 1 1 0 1 1 0 0 0 0 1 0 0 0 0 0 0 0]\n",
            " [0 1 1 1 0 0 0 0 0 0 0 0 1 1 0 0 1 0 0 1 1 1]\n",
            " [1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 1 0 0 0]\n",
            " [0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>bihday</th>\n",
              "      <th>cause</th>\n",
              "      <th>credit</th>\n",
              "      <th>dont</th>\n",
              "      <th>drags</th>\n",
              "      <th>dysfunction</th>\n",
              "      <th>dysfunctional</th>\n",
              "      <th>factsguide</th>\n",
              "      <th>father</th>\n",
              "      <th>kids</th>\n",
              "      <th>love</th>\n",
              "      <th>majesty</th>\n",
              "      <th>offer</th>\n",
              "      <th>pdx</th>\n",
              "      <th>selfish</th>\n",
              "      <th>society</th>\n",
              "      <th>thanks</th>\n",
              "      <th>time</th>\n",
              "      <th>ur</th>\n",
              "      <th>use</th>\n",
              "      <th>vans</th>\n",
              "      <th>wheelchair</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   bihday  cause  credit  dont  drags  ...  time  ur  use  vans  wheelchair\n",
              "0       0      0       0     0      1  ...     0   0    0     0           0\n",
              "1       0      1       1     1      0  ...     0   0    1     1           1\n",
              "2       1      0       0     0      0  ...     0   0    0     0           0\n",
              "3       0      0       0     0      0  ...     1   1    0     0           0\n",
              "4       0      0       0     0      0  ...     0   0    0     0           0\n",
              "\n",
              "[5 rows x 22 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M5j7gc4twmtH"
      },
      "source": [
        ""
      ],
      "execution_count": 64,
      "outputs": []
    }
  ]
}