review_sentiment5.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/KNU-BrainAI-Capstone2022/ITS/blob/main/review_sentiment5.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jcgr7Jw506xW",
        "outputId": "2ed2d908-f944-424c-90d1-11bf3bdb5fa3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: wordcloud in /usr/local/lib/python3.7/dist-packages (1.8.2.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from wordcloud) (3.2.2)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from wordcloud) (7.1.2)\n",
            "Requirement already satisfied: numpy>=1.6.1 in /usr/local/lib/python3.7/dist-packages (from wordcloud) (1.21.6)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->wordcloud) (3.0.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->wordcloud) (1.4.4)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->wordcloud) (0.11.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->wordcloud) (2.8.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib->wordcloud) (4.1.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib->wordcloud) (1.15.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.24.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.13.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.10.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n"
          ]
        }
      ],
      "source": [
        "!pip install wordcloud\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZfHT9fnx_gC",
        "outputId": "982ff5b3-50e5-4428-a278-f9f5353394fa"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "from transformers import BertTokenizer, BertForSequenceClassification\n",
        "\n",
        "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
        "import seaborn as sns"
      ],
      "metadata": {
        "id": "xY86_JA32zMk"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "print(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rAhti1Cg3AZs",
        "outputId": "dd49490f-ee73-42de-972d-dff1c25e6bf3"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "path = '/content/drive/MyDrive/processed_data.csv'\n",
        "\n",
        "df = pd.read_csv(path)\n",
        "print(df.shape)\n",
        "df.head()\n",
        "\n",
        "print(len(df.loc[df['rating'] > 3]))\n",
        "print(len(df.loc[df['rating'] < 3]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0RELDzs-3C5n",
        "outputId": "84585ed8-2c80-4ebe-d417-fca3013e88c9"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(8245, 5)\n",
            "4123\n",
            "4122\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df2 = df.loc[(df['label'] == 'OR') & (df['rating'] != 3) ]\n",
        "print(df2.shape)\n",
        "df2.head()\n",
        "print(len(df2.loc[df['rating'] > 3]))\n",
        "print(len(df2.loc[df['rating'] < 3]))\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_OYpMdt2iSxb",
        "outputId": "cec8e960-25db-4d0e-866b-b2f9bbd9d553"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(4162, 5)\n",
            "2065\n",
            "2097\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df2['division'] = np.select([df2.rating > 3], [1], default=0)\n",
        "df2[:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        },
        "id": "fm6a8OC33Eoc",
        "outputId": "9773cd84-404d-4115-c289-e9d4b32d3536"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    Unnamed: 0            category  rating label  \\\n",
              "7           64  Home_and_Kitchen_5       1    OR   \n",
              "8           67  Home_and_Kitchen_5       1    OR   \n",
              "9           87  Home_and_Kitchen_5       1    OR   \n",
              "10          95  Home_and_Kitchen_5       2    OR   \n",
              "11         105  Home_and_Kitchen_5       1    OR   \n",
              "\n",
              "                                                text_  division  \n",
              "7   Super rough, not soft wash cloths, more like b...         0  \n",
              "8      Does not do a very good job; difficult to use.         0  \n",
              "9   sheds all over, would not buy it ever again. b...         0  \n",
              "10   These are so weak. They barely bring in any air.         0  \n",
              "11  Stopped working after one use still waiting on...         0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-afab7079-ac83-47e2-bcb1-b9be7185282f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>category</th>\n",
              "      <th>rating</th>\n",
              "      <th>label</th>\n",
              "      <th>text_</th>\n",
              "      <th>division</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>64</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>1</td>\n",
              "      <td>OR</td>\n",
              "      <td>Super rough, not soft wash cloths, more like b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>67</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>1</td>\n",
              "      <td>OR</td>\n",
              "      <td>Does not do a very good job; difficult to use.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>87</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>1</td>\n",
              "      <td>OR</td>\n",
              "      <td>sheds all over, would not buy it ever again. b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>95</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>2</td>\n",
              "      <td>OR</td>\n",
              "      <td>These are so weak. They barely bring in any air.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>105</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>1</td>\n",
              "      <td>OR</td>\n",
              "      <td>Stopped working after one use still waiting on...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-afab7079-ac83-47e2-bcb1-b9be7185282f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-afab7079-ac83-47e2-bcb1-b9be7185282f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-afab7079-ac83-47e2-bcb1-b9be7185282f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-cased') # 'bert-base-cased': 영어용, 한국어 및 다른 언어시 변경 필요\n",
        "\n",
        "# example_text = 'I will watch Memento tonight'\n",
        "# bert_input = tokenizer(example_text,padding='max_length', max_length = 10, \n",
        "#                        truncation=True, return_tensors=\"pt\")\n",
        "\n",
        "# # padding : to pad each sequence to the maximum length that you specify\n",
        "# # max_length : the maximum length of each sequence.\n",
        "# #              For our actual dataset we will use 512, which is the maximum length of a sequence allowed for BERT.\n",
        "# # truncation : if True, then the tokens in each sequence that exceed the maximum length will be truncated.\n",
        "# # return_tensors : the type of tensors that will be returned. \n",
        "# #                  Pytorch: pt. \n",
        "# #                  Tensorflow: tf\n",
        "\n",
        "\n",
        "# print(bert_input['input_ids'])\n",
        "# # >>>tensor([[  101,   146,  1209,  2824,  2508, 26173,  3568,   102,     0,     0]])\n",
        "# # tokenizer.decode(bert_input.input_ids[0])\n",
        "# # >>> [CLS] I will watch Memento tonight [SEP] [PAD] [PAD]\n",
        "# # BertTokenizer가 알아서 다 해줌\n",
        "# print(bert_input['token_type_ids'])\n",
        "# # >>>tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
        "# print(bert_input['attention_mask'])\n",
        "# # >>>tensor([[1, 1, 1, 1, 1, 1, 1, 1, 0, 0]])"
      ],
      "metadata": {
        "id": "bzlzFfOA5FQv"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Dataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, df2):\n",
        "\n",
        "        self.divisions = [division for division in df2['division']]\n",
        "        self.texts = [tokenizer(text, \n",
        "                               padding='max_length', max_length = 128, truncation=True,\n",
        "                                return_tensors=\"pt\") for text in df2['text_']]\n",
        "    def classes(self):\n",
        "        return self.divisions\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.divisions)\n",
        "\n",
        "    def get_batch_divisions(self, idx):\n",
        "        # Fetch a batch of labels\n",
        "        return np.array(self.divisions[idx])\n",
        "\n",
        "    def get_batch_texts(self, idx):\n",
        "        # Fetch a batch of inputs\n",
        "        return self.texts[idx]\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "\n",
        "        batch_texts = self.get_batch_texts(idx)\n",
        "        batch_y = self.get_batch_divisions(idx)\n",
        "\n",
        "        return batch_texts, batch_y\n",
        "\n",
        "df2[:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "dN1nnSzb5TUJ",
        "outputId": "156b606f-576d-45bb-ae30-18b251d6f745"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    Unnamed: 0            category  rating label  \\\n",
              "7           64  Home_and_Kitchen_5       1    OR   \n",
              "8           67  Home_and_Kitchen_5       1    OR   \n",
              "9           87  Home_and_Kitchen_5       1    OR   \n",
              "10          95  Home_and_Kitchen_5       2    OR   \n",
              "11         105  Home_and_Kitchen_5       1    OR   \n",
              "\n",
              "                                                text_  division  \n",
              "7   Super rough, not soft wash cloths, more like b...         0  \n",
              "8      Does not do a very good job; difficult to use.         0  \n",
              "9   sheds all over, would not buy it ever again. b...         0  \n",
              "10   These are so weak. They barely bring in any air.         0  \n",
              "11  Stopped working after one use still waiting on...         0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-cb85b521-a637-4e8a-aeb2-794fa73e14ed\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>category</th>\n",
              "      <th>rating</th>\n",
              "      <th>label</th>\n",
              "      <th>text_</th>\n",
              "      <th>division</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>64</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>1</td>\n",
              "      <td>OR</td>\n",
              "      <td>Super rough, not soft wash cloths, more like b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>67</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>1</td>\n",
              "      <td>OR</td>\n",
              "      <td>Does not do a very good job; difficult to use.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>87</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>1</td>\n",
              "      <td>OR</td>\n",
              "      <td>sheds all over, would not buy it ever again. b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>95</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>2</td>\n",
              "      <td>OR</td>\n",
              "      <td>These are so weak. They barely bring in any air.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>105</td>\n",
              "      <td>Home_and_Kitchen_5</td>\n",
              "      <td>1</td>\n",
              "      <td>OR</td>\n",
              "      <td>Stopped working after one use still waiting on...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cb85b521-a637-4e8a-aeb2-794fa73e14ed')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-cb85b521-a637-4e8a-aeb2-794fa73e14ed button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-cb85b521-a637-4e8a-aeb2-794fa73e14ed');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.random.seed(112)\n",
        "df2_train, df2_val, df2_test = np.split(df2.sample(frac=1, random_state=42), \n",
        "                                     [int(.7*len(df2)), int(.9*len(df2))]) # train: 70, validation: 20, test: 10\n",
        "\n",
        "print(len(df2_train),len(df2_val), len(df2_test))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xse75DRj6Y-2",
        "outputId": "28e30db4-75b7-45f9-d2cf-9bf2e2601f2d"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2913 832 417\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(df2_train.loc[df['rating'] > 3]))\n",
        "print(len(df2_train.loc[df['rating'] < 3]))"
      ],
      "metadata": {
        "id": "gPMibGHedDHQ",
        "outputId": "9ecca20c-bc19-49f9-b7b7-ccf77c0927e6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1441\n",
            "1472\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from torch import nn\n",
        "from transformers import BertModel\n",
        "\n",
        "class BertClassifier(nn.Module):\n",
        "\n",
        "    def __init__(self, dropout=0.5):\n",
        "\n",
        "        super(BertClassifier, self).__init__()\n",
        "\n",
        "        self.bert = BertModel.from_pretrained('bert-base-cased') # 'bert-base-cased': 영어용, 다른 언어의 데이터면 다른거 쓰기\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.linear = nn.Linear(768, 2)\n",
        "        self.relu = nn.ReLU()\n",
        "\n",
        "    def forward(self, input_id, mask):\n",
        "\n",
        "        _, pooled_output = self.bert(input_ids= input_id, attention_mask=mask,return_dict=False)\n",
        "        # _: embedding vectors of all of the tokens in a sequence\n",
        "        # pooled_output: embedding vector of [CLS] token, 분류 문제에선 이것만 써도 충분\n",
        "        dropout_output = self.dropout(pooled_output)\n",
        "        linear_output = self.linear(dropout_output)\n",
        "        final_layer = self.relu(linear_output)\n",
        "\n",
        "        return final_layer"
      ],
      "metadata": {
        "id": "KsQdOBAj61Zo"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def saveModel(): \n",
        "    path = '/content/drive/MyDrive/NetModel_s.pth'\n",
        "    torch.save(model.state_dict(), path)"
      ],
      "metadata": {
        "id": "rUEHfQelbKFP"
      },
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "def train(model, train_data, val_data, learning_rate, epochs, cuda_switch):\n",
        "\n",
        "    train, val = Dataset(train_data), Dataset(val_data)\n",
        "\n",
        "    train_dataloader = torch.utils.data.DataLoader(train, batch_size=2, shuffle=True)\n",
        "    val_dataloader = torch.utils.data.DataLoader(val, batch_size=2)\n",
        "    best_accuracy = 0\n",
        "    \n",
        "    use_cuda = torch.cuda.is_available()\n",
        "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    optimizer = optim.Adam(model.parameters(), lr= learning_rate)\n",
        "\n",
        "    if cuda_switch:\n",
        "\n",
        "            model = model.cuda()\n",
        "            criterion = criterion.cuda()\n",
        "    \n",
        "\n",
        "    for epoch_num in range(epochs):\n",
        "\n",
        "            total_acc_train = 0\n",
        "            total_loss_train = 0\n",
        "            \n",
        "\n",
        "            for train_input, train_division in tqdm(train_dataloader):\n",
        "\n",
        "                train_division = train_division.to(device)\n",
        "                mask = train_input['attention_mask'].to(device)\n",
        "                input_id = train_input['input_ids'].squeeze(1).to(device)\n",
        "\n",
        "                output = model(input_id, mask)\n",
        "                \n",
        "                batch_loss = criterion(output, train_division)\n",
        "                total_loss_train += batch_loss.item()\n",
        "                \n",
        "                acc = (output.argmax(dim=1) == train_division).sum().item()\n",
        "                total_acc_train += acc\n",
        "\n",
        "                model.zero_grad()\n",
        "                batch_loss.backward()\n",
        "                optimizer.step()\n",
        "            \n",
        "            total_acc_val = 0\n",
        "            total_loss_val = 0\n",
        "\n",
        "            with torch.no_grad():\n",
        "\n",
        "                for val_input, val_division in val_dataloader:\n",
        "\n",
        "                    val_division = val_division.to(device)\n",
        "                    mask = val_input['attention_mask'].to(device)\n",
        "                    input_id = val_input['input_ids'].squeeze(1).to(device)\n",
        "\n",
        "                    output = model(input_id, mask)\n",
        "\n",
        "                    batch_loss = criterion(output, val_division)\n",
        "                    total_loss_val += batch_loss.item()\n",
        "                    \n",
        "                    acc = (output.argmax(dim=1) == val_division).sum().item()\n",
        "                    total_acc_val += acc\n",
        "            \n",
        "            print(\n",
        "                f'Epochs: {epoch_num + 1} | Train Loss: {total_loss_train / len(train_data): .3f} \\\n",
        "                | Train Accuracy: {total_acc_train / len(train_data): .3f} \\\n",
        "                | Val Loss: {total_loss_val / len(val_data): .3f} \\\n",
        "                | Val Accuracy: {total_acc_val / len(val_data): .3f}')\n",
        "            accuracy= (total_acc_val / len(val_data))\n",
        "\n",
        "            if accuracy > best_accuracy: \n",
        "               saveModel() \n",
        "               best_accuracy = accuracy \n",
        "\n",
        "EPOCHS = 10 \n",
        "model = BertClassifier()\n",
        "LR = 1e-6\n",
        "              \n",
        "train(model, df2_train, df2_val, LR, EPOCHS, use_cuda)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EOcXmOBm64kb",
        "outputId": "d966b11e-6b25-43a4-aeac-13d016cb5680"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "100%|██████████| 1457/1457 [02:06<00:00, 11.49it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 1 | Train Loss:  0.316                 | Train Accuracy:  0.638                 | Val Loss:  0.207                 | Val Accuracy:  0.870\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1457/1457 [02:03<00:00, 11.83it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 2 | Train Loss:  0.133                 | Train Accuracy:  0.921                 | Val Loss:  0.120                 | Val Accuracy:  0.921\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1457/1457 [02:02<00:00, 11.86it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 3 | Train Loss:  0.065                 | Train Accuracy:  0.968                 | Val Loss:  0.130                 | Val Accuracy:  0.900\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1457/1457 [02:01<00:00, 11.95it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 4 | Train Loss:  0.040                 | Train Accuracy:  0.983                 | Val Loss:  0.116                 | Val Accuracy:  0.929\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1457/1457 [02:03<00:00, 11.84it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 5 | Train Loss:  0.025                 | Train Accuracy:  0.990                 | Val Loss:  0.115                 | Val Accuracy:  0.927\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1457/1457 [02:01<00:00, 11.96it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 6 | Train Loss:  0.017                 | Train Accuracy:  0.994                 | Val Loss:  0.127                 | Val Accuracy:  0.927\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1457/1457 [02:02<00:00, 11.92it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 7 | Train Loss:  0.014                 | Train Accuracy:  0.995                 | Val Loss:  0.142                 | Val Accuracy:  0.922\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1457/1457 [02:02<00:00, 11.93it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 8 | Train Loss:  0.011                 | Train Accuracy:  0.995                 | Val Loss:  0.133                 | Val Accuracy:  0.928\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1457/1457 [02:02<00:00, 11.94it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 9 | Train Loss:  0.008                 | Train Accuracy:  0.995                 | Val Loss:  0.149                 | Val Accuracy:  0.924\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1457/1457 [02:01<00:00, 11.99it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 10 | Train Loss:  0.008                 | Train Accuracy:  0.994                 | Val Loss:  0.150                 | Val Accuracy:  0.929\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(model, test_data):\n",
        "\n",
        "    test = Dataset(test_data)\n",
        "\n",
        "    test_dataloader = torch.utils.data.DataLoader(test, batch_size=2)\n",
        "\n",
        "    use_cuda = torch.cuda.is_available()\n",
        "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "\n",
        "    if use_cuda:\n",
        "\n",
        "        model = model.cuda()\n",
        "\n",
        "    total_acc_test = 0\n",
        "    with torch.no_grad():\n",
        "\n",
        "        for test_input, test_division in test_dataloader:\n",
        "\n",
        "              test_division = test_division.to(device)\n",
        "              mask = test_input['attention_mask'].to(device)\n",
        "              input_id = test_input['input_ids'].squeeze(1).to(device)\n",
        "\n",
        "              output = model(input_id, mask)\n",
        "\n",
        "              acc = (output.argmax(dim=1) == test_division).sum().item()\n",
        "              total_acc_test += acc\n",
        "    \n",
        "    print(f'Test Accuracy: {total_acc_test / len(test_data): .3f}')\n",
        "    \n",
        "evaluate(model, df2_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "usg91Xjr7Jv3",
        "outputId": "63a8ab21-728f-4a13-db63-dde8316dee69"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test Accuracy:  0.945\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(model, test_data):\n",
        "\n",
        "    test = Dataset(test_data)\n",
        "\n",
        "    test_dataloader = torch.utils.data.DataLoader(test, batch_size=2)\n",
        "\n",
        "    use_cuda = torch.cuda.is_available()\n",
        "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "\n",
        "    if use_cuda:\n",
        "\n",
        "        model = model.cuda()\n",
        "\n",
        "    checking_num1 = 0\n",
        "    checking_num2 = 0\n",
        "    checking_num3 = 0\n",
        "    checking_num4 = 0\n",
        "\n",
        "    path = \"/content/drive/MyDrive/NetModel_s.pth\"\n",
        "    model.load_state_dict(torch.load(path))\n",
        "    total_acc_test = 0\n",
        "    with torch.no_grad():\n",
        "\n",
        "        for test_input, test_label in test_dataloader:\n",
        "\n",
        "              test_label = test_label.to(device)\n",
        "              mask = test_input['attention_mask'].to(device)\n",
        "              input_id = test_input['input_ids'].squeeze(1).to(device)\n",
        "\n",
        "              output = model(input_id, mask)\n",
        "              if output.argmax(dim=1)[0] == 1:\n",
        "                  checking_num1 +=1\n",
        "              elif output.argmax(dim=1)[0] == 0:\n",
        "                  checking_num3 +=1\n",
        "              else:\n",
        "                  checking_num4 +=1\n",
        "              checking_num2+=1\n",
        "              acc = (output.argmax(dim=1) == test_label).sum().item()\n",
        "              total_acc_test += acc\n",
        "    \n",
        "    print(f'Test Accuracy: {total_acc_test / len(test_data): .3f}')\n",
        "    print('==',total_acc_test)\n",
        "    print('==',len(test_data))\n",
        "    print(checking_num1)\n",
        "    print(checking_num2)\n",
        "    print(checking_num3)\n",
        "    print(checking_num4)\n",
        "    print(f'Positive review : ', checking_num1)\n",
        "    print(f'Total review : ', checking_num2)\n",
        "    print(f'Positive review rate: {(checking_num1/checking_num2)*100: .3f}')\n",
        "evaluate(model, df2_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y6_VMXqSnJ_Z",
        "outputId": "742e5e03-c56d-4e6c-9114-460abfbfd4f9"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test Accuracy:  0.940\n",
            "== 392\n",
            "== 417\n",
            "106\n",
            "209\n",
            "103\n",
            "0\n",
            "Positive review :  106\n",
            "Total review :  209\n",
            "Positive review rate:  50.718\n"
          ]
        }
      ]
    }
  ]
}