Opinion_summarization_TPU_T5.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "name": "Copy of Opinion_summarization_TPU_T5",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ca47c0adf2ad47f68bd47f8caedaea7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_271acc872cc14023a0cf22ea2e1d8901",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_33698875d79e400098b4eafe018ecc94",
              "IPY_MODEL_a094d2d7f5454257999da65a9066fefc"
            ]
          }
        },
        "271acc872cc14023a0cf22ea2e1d8901": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "33698875d79e400098b4eafe018ecc94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d7b222194c0c4ef59441307b68de47fe",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 791656,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791656,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_718a1438af0a49ceb6ec956a2c5b7a61"
          }
        },
        "a094d2d7f5454257999da65a9066fefc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9514cd056e2149cc86be00881664b748",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 792k/792k [00:11&lt;00:00, 68.9kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_345cbd4c4cf64a85933a1538f76036a1"
          }
        },
        "d7b222194c0c4ef59441307b68de47fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "718a1438af0a49ceb6ec956a2c5b7a61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9514cd056e2149cc86be00881664b748": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "345cbd4c4cf64a85933a1538f76036a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a91281b20bf54b1689699bfa0331cb85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c916817a6c68440e971965f6bb6b60ac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c342fa49fb294f229c13bfaa7131ef56",
              "IPY_MODEL_f0154700444e44e18f5d81a20115d9b1"
            ]
          }
        },
        "c916817a6c68440e971965f6bb6b60ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c342fa49fb294f229c13bfaa7131ef56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_54e3e4e2130b49358bd086c654fa941e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1197,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1197,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bb8a7e0c7c4c433ebecd16ca88139919"
          }
        },
        "f0154700444e44e18f5d81a20115d9b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e1bccb01bed34a95aa201adda5cf2c84",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.20k/1.20k [00:00&lt;00:00, 3.88kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b05d1bf6f84840ebb43c9687641630e8"
          }
        },
        "54e3e4e2130b49358bd086c654fa941e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bb8a7e0c7c4c433ebecd16ca88139919": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e1bccb01bed34a95aa201adda5cf2c84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b05d1bf6f84840ebb43c9687641630e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "SAzIwJg2vweI"
      },
      "source": [
        "import os\n",
        "assert os.environ['COLAB_TPU_ADDR'], 'Make sure to select TPU from Edit > Notebook settings > Hardware accelerator'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "197oSNuasSgT",
        "outputId": "aa12fae7-8c93-4a06-f857-460e46e75c65"
      },
      "source": [
        "# installing required libraries and downloading data from source\n",
        "\n",
        "!pip3 install transformers -q\n",
        "!pip3 install rouge -q\n",
        "!pip3 install sentencepiece\n",
        "!wget https://opiniondigest.s3-us-west-2.amazonaws.com/data/yelp-default-data.zip -q\n",
        "!unzip yelp-default-data.zip\n",
        "!rm yelp-default-data.zip\n",
        "!pip install cloud-tpu-client==0.10 https://storage.googleapis.com/tpu-pytorch/wheels/torch_xla-1.7-cp36-cp36m-linux_x86_64.whl -q"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.5MB 7.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 21.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.9MB 27.9MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Archive:  yelp-default-data.zip\n",
            "   creating: data/\n",
            "   creating: data/yelp-default/\n",
            "  inflating: data/yelp-default/train.csv  \n",
            "  inflating: data/yelp-default/test_gold_8_15_all_all_300_8.csv  \n",
            "  inflating: data/yelp-default/summaries_0-200_cleaned_fixed_business_ids.csv  \n",
            "  inflating: data/yelp-default/test.csv  \n",
            "  inflating: data/yelp-default/yelp.jsonl  \n",
            "  inflating: data/yelp-default/dev.csv  \n",
            "  inflating: data/yelp-default/test_gold.csv  \n",
            "\u001b[K     |████████████████████████████████| 133.6MB 30kB/s \n",
            "\u001b[K     |████████████████████████████████| 61kB 3.4MB/s \n",
            "\u001b[?25h"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8m_DBMRoIKdz"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from transformers import T5Tokenizer, EncoderDecoderModel, T5Model, T5ForConditionalGeneration\n",
        "import torch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6QafAZmYXUq-"
      },
      "source": [
        "# reading data and making required changes to the files\n",
        "\n",
        "train = pd.read_csv(\"/content/data/yelp-default/train.csv\").loc[:]\n",
        "val = pd.read_csv(\"/content/data/yelp-default/dev.csv\").loc[:]\n",
        "train.columns = ['eid', 'rid', 'text', 'extraction', 'phrases']\n",
        "val.columns = ['eid', 'rid', 'text', 'extraction', 'phrases']\n",
        "train.to_csv('./train.csv')\n",
        "val.to_csv('./val.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 117,
          "referenced_widgets": [
            "ca47c0adf2ad47f68bd47f8caedaea7b",
            "271acc872cc14023a0cf22ea2e1d8901",
            "33698875d79e400098b4eafe018ecc94",
            "a094d2d7f5454257999da65a9066fefc",
            "d7b222194c0c4ef59441307b68de47fe",
            "718a1438af0a49ceb6ec956a2c5b7a61",
            "9514cd056e2149cc86be00881664b748",
            "345cbd4c4cf64a85933a1538f76036a1"
          ]
        },
        "id": "qZklrQoaQlYH",
        "outputId": "fb3db401-cee0-4e99-87de-18c3ce9deae9"
      },
      "source": [
        "# config class contains all the required configurations for the training of models\n",
        "\n",
        "class config:\n",
        "\n",
        "    MAX_LEN = 128\n",
        "    # TOKENIZER = transformers.BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "    TOKENIZER = T5Tokenizer.from_pretrained('t5-small')\n",
        "    MODEL_LIST = ['bert-base-uncased']\n",
        "\n",
        "    # Model parameters\n",
        "    BATCH_SIZE = 16\n",
        "    SHUFFLE = False\n",
        "    NO_OF_WORKERS = 1\n",
        "    EPOCHS = 100\n",
        "    LR = 1e-2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:Waiting for TPU to be start up with version pytorch-1.7...\n",
            "WARNING:root:Waiting for TPU to be start up with version pytorch-1.7...\n",
            "WARNING:root:TPU has started up successfully with version pytorch-1.7\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ca47c0adf2ad47f68bd47f8caedaea7b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=791656.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hDOLFIzQ2D0"
      },
      "source": [
        "# dataset class contains data preparation functions \n",
        "\n",
        "class dataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, data):\n",
        "        self.tokenizer = config.TOKENIZER\n",
        "        self.max_len = config.MAX_LEN\n",
        "        self.data = data\n",
        "\n",
        "    def get_target(self, data):\n",
        "        text = data['text']\n",
        "        phrases = 'summarize: ' + ','.join(np.random.permutation(data['phrases'].split(' [SEP] ')))\n",
        "        # phrases = 'summarize: ' + data['phrases'].replace(' [SEP] ', ',')\n",
        "\n",
        "\n",
        "        encoded_text = self.tokenizer.encode_plus(\n",
        "            text,\n",
        "            truncation=True,\n",
        "            max_length=self.max_len,\n",
        "            add_special_tokens=True,\n",
        "            padding = 'max_length',\n",
        "            return_attention_mask=True,\n",
        "            return_token_type_ids=True,\n",
        "            return_tensors='pt',\n",
        "        )\n",
        "\n",
        "        encoded_phrases = self.tokenizer.encode_plus(\n",
        "            phrases,\n",
        "            truncation=True,\n",
        "            max_length=self.max_len,\n",
        "            add_special_tokens=True,\n",
        "            padding = 'max_length',\n",
        "            return_attention_mask=True,\n",
        "            return_token_type_ids=True,\n",
        "            return_tensors='pt',\n",
        "        )\n",
        "\n",
        "        input_ids = encoded_text.input_ids[0]\n",
        "        token_type_ids = encoded_text.token_type_ids[0]\n",
        "        attention_mask = encoded_text.attention_mask[0]\n",
        "\n",
        "        p_input_ids = encoded_phrases.input_ids[0]\n",
        "        p_token_type_ids = encoded_phrases.token_type_ids[0]\n",
        "        p_attention_mask = encoded_phrases.attention_mask[0]\n",
        "\n",
        "        return {\"review\": text, \"input_ids\": input_ids, \"token_type_ids\": token_type_ids, \"attention_mask\": attention_mask,\n",
        "                \"phrases\": phrases, \"p_input_ids\": p_input_ids, \"p_token_type_ids\": p_token_type_ids, \"p_attention_mask\": p_attention_mask\n",
        "                }\n",
        "\n",
        "    def __len__(self):\n",
        "        'Denotes the total number of samples'\n",
        "        return len(self.data)\n",
        "\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        return self.get_target(self.data.iloc[index])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fqqgT1RPQ6gY"
      },
      "source": [
        "# function to define models \n",
        "\n",
        "def get_model(model=0):\n",
        "    # return EncoderDecoderModel.from_encoder_decoder_pretrained(config.MODEL_LIST[model], config.MODEL_LIST[model])\n",
        "    return T5ForConditionalGeneration.from_pretrained('t5-small')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X5IfkCUbXUuW"
      },
      "source": [
        "# function to train on TPUs\n",
        "\n",
        "from tqdm import tqdm\n",
        "import transformers as tr\n",
        "from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel, T5Config, T5ForConditionalGeneration\n",
        "import torch.optim as optim\n",
        "import torch_xla\n",
        "import torch_xla.core.xla_model as xm\n",
        "import torch_xla.distributed.xla_multiprocessing as xmp\n",
        "import torch_xla.distributed.parallel_loader as pl\n",
        "import time\n",
        "\n",
        "def map_fn(index, flags):\n",
        "    torch.manual_seed(flags['seed'])\n",
        "\n",
        "    \n",
        "    device = xm.xla_device()  \n",
        "\n",
        "\n",
        "    print(\"Process\", index ,\"is using\", xm.xla_real_devices([str(device)])[0])\n",
        "\n",
        "    train_dataset = dataset(pd.read_csv('./train.csv'))\n",
        "    val_dataset = dataset(pd.read_csv('./val.csv'))\n",
        "\n",
        "    \n",
        "    train_sampler = torch.utils.data.distributed.DistributedSampler(\n",
        "        train_dataset,\n",
        "        num_replicas=xm.xrt_world_size(),\n",
        "        rank=xm.get_ordinal(),\n",
        "        shuffle=True)\n",
        "    \n",
        "    val_sampler = torch.utils.data.distributed.DistributedSampler(\n",
        "        val_dataset,\n",
        "        num_replicas=xm.xrt_world_size(),\n",
        "        rank=xm.get_ordinal(),\n",
        "        shuffle=False)\n",
        "    \n",
        "    train_loader = torch.utils.data.DataLoader(\n",
        "        train_dataset,\n",
        "        batch_size=flags['batch_size'],\n",
        "        sampler=train_sampler,\n",
        "        num_workers=flags['num_workers'],\n",
        "        drop_last=True)\n",
        "\n",
        "    val_loader = torch.utils.data.DataLoader(\n",
        "        val_dataset,\n",
        "        batch_size=flags['batch_size'],\n",
        "        sampler=val_sampler,\n",
        "        shuffle=False,\n",
        "        num_workers=flags['num_workers'],\n",
        "        drop_last=True)\n",
        "    \n",
        "    #load saved model\n",
        "    # config_encoder = BertConfig()\n",
        "    # config_decoder = BertConfig()\n",
        "\n",
        "    # model_config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)\n",
        "    # model = tr.EncoderDecoderModel.from_pretrained(\"/content/drive/My Drive/bert_model_final_training_epoch_1\",config=model_config).to(device).train()\n",
        "\n",
        "    model_config = T5Config.from_pretrained('t5-small')\n",
        "    model = T5ForConditionalGeneration.from_pretrained('/content/drive/MyDrive/t5_small_10epoch_ALL', config=model_config).to(device).train()\n",
        "\n",
        "    # model = get_model().to(device).train()\n",
        "    optimizer = optim.Adam(model.parameters(), lr = flags['lr'])\n",
        "    \n",
        "    train_start = time.time()\n",
        "    print(\"training started\")\n",
        "    for epoch in range(flags['num_epochs']):\n",
        "        para_train_loader = pl.ParallelLoader(train_loader, [device]).per_device_loader(device)\n",
        "        epoch_loss = 0\n",
        "        for batch_num, batch in enumerate(tqdm(para_train_loader)):\n",
        "\n",
        "            de_output = batch['input_ids']\n",
        "            de_attention_mask = batch['attention_mask']\n",
        "\n",
        "            p_input_ids = batch['p_input_ids']\n",
        "            p_attention_mask = batch['p_attention_mask']\n",
        "\n",
        "            lm_labels = de_output.clone()\n",
        "\n",
        "            output = model(input_ids=p_input_ids, attention_mask=p_attention_mask,labels = lm_labels)\n",
        "\n",
        "            loss = output[0]\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            loss.backward()\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "            xm.optimizer_step(optimizer)\n",
        "\n",
        "            # scheduler.step()\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "        \n",
        "        print(\"Mean epoch loss:\", (epoch_loss / (batch_num+1)))\n",
        "\n",
        "\n",
        "    elapsed_train_time = time.time() - train_start\n",
        "    print(\"Process\", index, \"finished training. Train time was:\", elapsed_train_time) \n",
        "\n",
        "    model.eval()\n",
        "    eval_start = time.time()\n",
        "    with torch.no_grad():\n",
        "        val_loss = 0\n",
        "\n",
        "        para_val_loader = pl.ParallelLoader(val_loader, [device]).per_device_loader(device)\n",
        "        for batch_num, batch in enumerate(tqdm(para_val_loader)):\n",
        "            de_output = batch['input_ids']\n",
        "            de_attention_mask = batch['attention_mask']\n",
        "\n",
        "            p_input_ids = batch['p_input_ids']\n",
        "            p_attention_mask = batch['p_attention_mask']\n",
        "\n",
        "            lm_labels = de_output.clone()\n",
        "\n",
        "            output = model(input_ids=p_input_ids, attention_mask=p_attention_mask,labels = lm_labels)\n",
        "\n",
        "            loss = output[0]\n",
        "            val_loss += loss.item()\n",
        "        \n",
        "        print(\"Mean val loss:\", (val_loss / (batch_num+1)))\n",
        "\n",
        "    xm.save(model.state_dict(), '/content/drive/MyDrive/t5_small_10epoch_ALL')\n",
        "    elapsed_eval_time = time.time() - eval_start\n",
        "    print(\"Process\", index, \"finished evaluation. Evaluation time was:\", elapsed_eval_time)\n",
        "    print(\"Process\", index, \"Mean eval_loss\", (val_loss / (batch_num+1)))\n",
        "    print('MODEL SAVED!')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fCeDN9wSXnRs",
        "outputId": "70df0ac2-bdba-4f39-c166-4c7e10442747"
      },
      "source": [
        "%%time\n",
        "flags = {}\n",
        "flags['lr'] = 1e-3\n",
        "flags['batch_size'] = 16\n",
        "flags['num_workers'] = 8\n",
        "flags['num_epochs'] = 1\n",
        "flags['seed'] = 8888\n",
        "# flags['model'] = 1\n",
        "\n",
        "xmp.spawn(map_fn, args = (flags,), nprocs = 8, start_method = 'fork')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Process 0 is using TPU:0\n",
            "Process 2 is using TPU:2\n",
            "Process 1 is using TPU:1\n",
            "Process 5 is using TPU:5\n",
            "Process 6 is using TPU:6\n",
            "Process 3 is using TPU:3\n",
            "Process 7 is using TPU:7\n",
            "Process 4 is using TPU:4\n",
            "training started\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/4875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "training started\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/4875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "training started\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/4875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "training started\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/4875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "training started\n",
            "training started\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/4875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "training started\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/4875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "training started\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 4875/4875 [22:01<00:00,  3.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean epoch loss: 1.6336855308581621\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 4875/4875 [22:01<00:00,  3.69it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Process 6 finished training. Train time was: 1321.999601840973\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean epoch loss: 1.6246947194612944\n",
            "Process 5 finished training. Train time was: 1321.998363494873\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 4875/4875 [22:12<00:00,  3.66it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean epoch loss: 1.6252446906016422\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 4875/4875 [22:01<00:00,  4.03it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Process 2 finished training. Train time was: 1332.1820726394653\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 4875/4875 [22:16<00:00,  3.65it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean epoch loss: 1.626094518783765\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 4875/4875 [22:02<00:00,  3.69it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Process 0 finished training. Train time was: 1336.4975697994232\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean epoch loss: 1.6299827803709568\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 4875/4875 [21:59<00:00,  3.99it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Process 3 finished training. Train time was: 1322.503359079361\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 4875/4875 [22:00<00:00,  3.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean epoch loss: 1.6306831300442035\n",
            "Process 4 finished training. Train time was: 1320.9594633579254\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 4875/4875 [22:01<00:00,  3.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean epoch loss: 1.6235088771428818\n",
            "Process 7 finished training. Train time was: 1322.1467015743256\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 4875/4875 [22:00<00:00,  3.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean epoch loss: 1.6298897025768573\n",
            "Process 1 finished training. Train time was: 1320.1336843967438\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 603/603 [00:56<00:00, 10.59it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean val loss: 1.6203955665947392\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 603/603 [00:56<00:00, 10.62it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean val loss: 1.6106466928722452\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 603/603 [00:56<00:00, 10.61it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean val loss: 1.614912199539134\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 603/603 [00:57<00:00, 10.57it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean val loss: 1.5915811197279301\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 603/603 [00:57<00:00, 10.53it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean val loss: 1.606997630094019\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 603/603 [00:57<00:00, 10.56it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean val loss: 1.622126676549959\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r100%|██████████| 603/603 [00:57<00:00, 10.55it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean val loss: 1.6230337538727084\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 603/603 [00:57<00:00, 10.53it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Mean val loss: 1.6118681824227075\n",
            "Process 7 finished evaluation. Evaluation time was: 60.936707735061646\n",
            "Process 5 finished evaluation. Evaluation time was: 61.10967707633972\n",
            "Process 2 finished evaluation. Evaluation time was: 61.10675263404846\n",
            "Process 6 finished evaluation. Evaluation time was: 61.11290907859802\n",
            "Process 1 finished evaluation. Evaluation time was: 60.93226218223572\n",
            "Process 3 finished evaluation. Evaluation time was: 61.05033016204834\n",
            "Process 4 finished evaluation. Evaluation time was: 61.01758551597595\n",
            "Process 7 Mean eval_loss 1.6230337538727084\n",
            "Process 5 Mean eval_loss 1.614912199539134\n",
            "Process 2 Mean eval_loss 1.6118681824227075\n",
            "Process 6 Mean eval_loss 1.6203955665947392\n",
            "Process 4 Mean eval_loss 1.622126676549959\n",
            "Process 1 Mean eval_loss 1.6106466928722452\n",
            "Process 3 Mean eval_loss 1.606997630094019\n",
            "MODEL SAVED!\n",
            "MODEL SAVED!\n",
            "MODEL SAVED!\n",
            "MODEL SAVED!\n",
            "MODEL SAVED!\n",
            "MODEL SAVED!\n",
            "MODEL SAVED!\n",
            "Process 0 finished evaluation. Evaluation time was: 61.09617590904236\n",
            "Process 0 Mean eval_loss 1.5915811197279301\n",
            "MODEL SAVED!\n",
            "CPU times: user 18.9 s, sys: 12.1 s, total: 31 s\n",
            "Wall time: 23min 45s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EekGozyBMCMm"
      },
      "source": [
        "# train_dataset=dataset(pd.read_csv('./train.csv'))\n",
        "# for data in train_dataset:\n",
        "#     print(data)\n",
        "#     break"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "U5-pauekMWKs",
        "outputId": "5c9d7992-ef51-4a0a-b55c-35ca5db3fdbe"
      },
      "source": [
        "# config.TOKENIZER.decode([6])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "','"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NzSC_-KE3c2I"
      },
      "source": [
        "    BERT2BERT 60k 10 epochs 5e-5 16 -> 1.7     -> 27 9.9 25.9\n",
        "    ROBERTA2ROBERTA 60k 10 epochs 5e-5 16 -> "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fKkAXoujXy8h",
        "outputId": "77f7a118-45d0-43c5-d0a5-eccca09c1a14"
      },
      "source": [
        "!pip install sumeval\n",
        "import transformers as tr\n",
        "# from rouge import Rouge\n",
        "from tqdm import tqdm\n",
        "# rouge = Rouge(metrics = ['rouge-1','rouge-2', 'rouge-l'])\n",
        "from sumeval.metrics.rouge import RougeCalculator\n",
        "rouge = RougeCalculator(stopwords = False, lang = \"en\")\n",
        "\n",
        "def evaluateSummary(hypothesis, reference) :\t\n",
        "    size = len(hypothesis)\n",
        "    rouge_1 = 0\n",
        "    rouge_2 = 0\n",
        "    rouge_l = 0\n",
        "\n",
        "    for i in range(size) :\n",
        "        rouge_1 = rouge_1 + rouge.rouge_n(hypothesis[i], reference[i], n=1)\n",
        "        rouge_2 = rouge_2 + rouge.rouge_n(hypothesis[i], reference[i], n=2)\n",
        "        rouge_l = rouge_l + rouge.rouge_l(hypothesis[i], reference[i])\n",
        "\n",
        "    avg_score = {'Rouge-1' : rouge_1/size,\n",
        "    \t        'Rouge-2' : rouge_2/size,\n",
        "                'Rouge-L' : rouge_l/size,\n",
        "        }\t    \n",
        "    return avg_score"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sumeval\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e6/87/bfc0f9397b9421305863edfdd2dbea637e47204976cb5473535c856338f4/sumeval-0.2.2.tar.gz (80kB)\n",
            "\r\u001b[K     |████                            | 10kB 18.4MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 20kB 14.4MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 30kB 10.4MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 40kB 9.1MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 51kB 5.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 61kB 5.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 71kB 6.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 81kB 4.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: plac>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from sumeval) (1.1.3)\n",
            "Collecting sacrebleu>=1.3.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/c4/8e948f601a4f9609e8b2b58f31966cb13cf17b940b82aa3e767f01c42c52/sacrebleu-1.4.14-py3-none-any.whl (64kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 8.1MB/s \n",
            "\u001b[?25hCollecting portalocker\n",
            "  Downloading https://files.pythonhosted.org/packages/89/a6/3814b7107e0788040870e8825eebf214d72166adf656ba7d4bf14759a06a/portalocker-2.0.0-py2.py3-none-any.whl\n",
            "Building wheels for collected packages: sumeval\n",
            "  Building wheel for sumeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sumeval: filename=sumeval-0.2.2-cp36-none-any.whl size=54537 sha256=14907c07cbc536f4323caffbfbcd36ad2e3a79240c931cd58336d0105f44c42b\n",
            "  Stored in directory: /root/.cache/pip/wheels/7b/6f/57/19ceecab21445c88f3c565735fa1887b4cd18d340c972eb445\n",
            "Successfully built sumeval\n",
            "Installing collected packages: portalocker, sacrebleu, sumeval\n",
            "Successfully installed portalocker-2.0.0 sacrebleu-1.4.14 sumeval-0.2.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "a91281b20bf54b1689699bfa0331cb85",
            "c916817a6c68440e971965f6bb6b60ac",
            "c342fa49fb294f229c13bfaa7131ef56",
            "f0154700444e44e18f5d81a20115d9b1",
            "54e3e4e2130b49358bd086c654fa941e",
            "bb8a7e0c7c4c433ebecd16ca88139919",
            "e1bccb01bed34a95aa201adda5cf2c84",
            "b05d1bf6f84840ebb43c9687641630e8"
          ]
        },
        "id": "pPuDU2KpGycj",
        "outputId": "3b6bfbbb-2d82-4a80-d51a-a6ea7013515f"
      },
      "source": [
        "# import torch_xla\n",
        "# import torch_xla.core.xla_model as xm\n",
        "\n",
        "# dev=xm.xla_device()\n",
        "# dev='cuda'\n",
        "import transformers\n",
        "from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel, RobertaConfig, T5Config, T5Tokenizer, T5ForConditionalGeneration\n",
        "import torch\n",
        "import pandas as pd\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "# config_encoder = RobertaConfig.from_pretrained('roberta-base')\n",
        "# config_decoder = RobertaConfig.from_pretrained('roberta-base')\n",
        "\n",
        "# config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)\n",
        "# model = tr.EncoderDecoderModel.from_pretrained(\"./my_model\",config=config).to(device).eval()\n",
        "\n",
        "# tok=tr.BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "# tok = tr.RobertaTokenizer.from_pretrained('roberta-base')\n",
        "\n",
        "model_config = T5Config.from_pretrained('t5-small')\n",
        "model = T5ForConditionalGeneration.from_pretrained('/content/drive/MyDrive/t5_small_10epoch_ALL',config=model_config).to(device).eval()\n",
        "\n",
        "tok = T5Tokenizer.from_pretrained('t5-small')\n",
        "\n",
        "model.num_parameters()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a91281b20bf54b1689699bfa0331cb85",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1197.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "60506624"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UQv7jWc4QQ7K",
        "outputId": "ac82dedf-1ffe-4e5a-e46a-b6683e152184"
      },
      "source": [
        "model.state_dict().keys()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "odict_keys(['shared.weight', 'encoder.embed_tokens.weight', 'encoder.block.0.layer.0.SelfAttention.q.weight', 'encoder.block.0.layer.0.SelfAttention.k.weight', 'encoder.block.0.layer.0.SelfAttention.v.weight', 'encoder.block.0.layer.0.SelfAttention.o.weight', 'encoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight', 'encoder.block.0.layer.0.layer_norm.weight', 'encoder.block.0.layer.1.DenseReluDense.wi.weight', 'encoder.block.0.layer.1.DenseReluDense.wo.weight', 'encoder.block.0.layer.1.layer_norm.weight', 'encoder.block.1.layer.0.SelfAttention.q.weight', 'encoder.block.1.layer.0.SelfAttention.k.weight', 'encoder.block.1.layer.0.SelfAttention.v.weight', 'encoder.block.1.layer.0.SelfAttention.o.weight', 'encoder.block.1.layer.0.layer_norm.weight', 'encoder.block.1.layer.1.DenseReluDense.wi.weight', 'encoder.block.1.layer.1.DenseReluDense.wo.weight', 'encoder.block.1.layer.1.layer_norm.weight', 'encoder.block.2.layer.0.SelfAttention.q.weight', 'encoder.block.2.layer.0.SelfAttention.k.weight', 'encoder.block.2.layer.0.SelfAttention.v.weight', 'encoder.block.2.layer.0.SelfAttention.o.weight', 'encoder.block.2.layer.0.layer_norm.weight', 'encoder.block.2.layer.1.DenseReluDense.wi.weight', 'encoder.block.2.layer.1.DenseReluDense.wo.weight', 'encoder.block.2.layer.1.layer_norm.weight', 'encoder.block.3.layer.0.SelfAttention.q.weight', 'encoder.block.3.layer.0.SelfAttention.k.weight', 'encoder.block.3.layer.0.SelfAttention.v.weight', 'encoder.block.3.layer.0.SelfAttention.o.weight', 'encoder.block.3.layer.0.layer_norm.weight', 'encoder.block.3.layer.1.DenseReluDense.wi.weight', 'encoder.block.3.layer.1.DenseReluDense.wo.weight', 'encoder.block.3.layer.1.layer_norm.weight', 'encoder.block.4.layer.0.SelfAttention.q.weight', 'encoder.block.4.layer.0.SelfAttention.k.weight', 'encoder.block.4.layer.0.SelfAttention.v.weight', 'encoder.block.4.layer.0.SelfAttention.o.weight', 'encoder.block.4.layer.0.layer_norm.weight', 'encoder.block.4.layer.1.DenseReluDense.wi.weight', 'encoder.block.4.layer.1.DenseReluDense.wo.weight', 'encoder.block.4.layer.1.layer_norm.weight', 'encoder.block.5.layer.0.SelfAttention.q.weight', 'encoder.block.5.layer.0.SelfAttention.k.weight', 'encoder.block.5.layer.0.SelfAttention.v.weight', 'encoder.block.5.layer.0.SelfAttention.o.weight', 'encoder.block.5.layer.0.layer_norm.weight', 'encoder.block.5.layer.1.DenseReluDense.wi.weight', 'encoder.block.5.layer.1.DenseReluDense.wo.weight', 'encoder.block.5.layer.1.layer_norm.weight', 'encoder.final_layer_norm.weight', 'decoder.embed_tokens.weight', 'decoder.block.0.layer.0.SelfAttention.q.weight', 'decoder.block.0.layer.0.SelfAttention.k.weight', 'decoder.block.0.layer.0.SelfAttention.v.weight', 'decoder.block.0.layer.0.SelfAttention.o.weight', 'decoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight', 'decoder.block.0.layer.0.layer_norm.weight', 'decoder.block.0.layer.1.EncDecAttention.q.weight', 'decoder.block.0.layer.1.EncDecAttention.k.weight', 'decoder.block.0.layer.1.EncDecAttention.v.weight', 'decoder.block.0.layer.1.EncDecAttention.o.weight', 'decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight', 'decoder.block.0.layer.1.layer_norm.weight', 'decoder.block.0.layer.2.DenseReluDense.wi.weight', 'decoder.block.0.layer.2.DenseReluDense.wo.weight', 'decoder.block.0.layer.2.layer_norm.weight', 'decoder.block.1.layer.0.SelfAttention.q.weight', 'decoder.block.1.layer.0.SelfAttention.k.weight', 'decoder.block.1.layer.0.SelfAttention.v.weight', 'decoder.block.1.layer.0.SelfAttention.o.weight', 'decoder.block.1.layer.0.layer_norm.weight', 'decoder.block.1.layer.1.EncDecAttention.q.weight', 'decoder.block.1.layer.1.EncDecAttention.k.weight', 'decoder.block.1.layer.1.EncDecAttention.v.weight', 'decoder.block.1.layer.1.EncDecAttention.o.weight', 'decoder.block.1.layer.1.layer_norm.weight', 'decoder.block.1.layer.2.DenseReluDense.wi.weight', 'decoder.block.1.layer.2.DenseReluDense.wo.weight', 'decoder.block.1.layer.2.layer_norm.weight', 'decoder.block.2.layer.0.SelfAttention.q.weight', 'decoder.block.2.layer.0.SelfAttention.k.weight', 'decoder.block.2.layer.0.SelfAttention.v.weight', 'decoder.block.2.layer.0.SelfAttention.o.weight', 'decoder.block.2.layer.0.layer_norm.weight', 'decoder.block.2.layer.1.EncDecAttention.q.weight', 'decoder.block.2.layer.1.EncDecAttention.k.weight', 'decoder.block.2.layer.1.EncDecAttention.v.weight', 'decoder.block.2.layer.1.EncDecAttention.o.weight', 'decoder.block.2.layer.1.layer_norm.weight', 'decoder.block.2.layer.2.DenseReluDense.wi.weight', 'decoder.block.2.layer.2.DenseReluDense.wo.weight', 'decoder.block.2.layer.2.layer_norm.weight', 'decoder.block.3.layer.0.SelfAttention.q.weight', 'decoder.block.3.layer.0.SelfAttention.k.weight', 'decoder.block.3.layer.0.SelfAttention.v.weight', 'decoder.block.3.layer.0.SelfAttention.o.weight', 'decoder.block.3.layer.0.layer_norm.weight', 'decoder.block.3.layer.1.EncDecAttention.q.weight', 'decoder.block.3.layer.1.EncDecAttention.k.weight', 'decoder.block.3.layer.1.EncDecAttention.v.weight', 'decoder.block.3.layer.1.EncDecAttention.o.weight', 'decoder.block.3.layer.1.layer_norm.weight', 'decoder.block.3.layer.2.DenseReluDense.wi.weight', 'decoder.block.3.layer.2.DenseReluDense.wo.weight', 'decoder.block.3.layer.2.layer_norm.weight', 'decoder.block.4.layer.0.SelfAttention.q.weight', 'decoder.block.4.layer.0.SelfAttention.k.weight', 'decoder.block.4.layer.0.SelfAttention.v.weight', 'decoder.block.4.layer.0.SelfAttention.o.weight', 'decoder.block.4.layer.0.layer_norm.weight', 'decoder.block.4.layer.1.EncDecAttention.q.weight', 'decoder.block.4.layer.1.EncDecAttention.k.weight', 'decoder.block.4.layer.1.EncDecAttention.v.weight', 'decoder.block.4.layer.1.EncDecAttention.o.weight', 'decoder.block.4.layer.1.layer_norm.weight', 'decoder.block.4.layer.2.DenseReluDense.wi.weight', 'decoder.block.4.layer.2.DenseReluDense.wo.weight', 'decoder.block.4.layer.2.layer_norm.weight', 'decoder.block.5.layer.0.SelfAttention.q.weight', 'decoder.block.5.layer.0.SelfAttention.k.weight', 'decoder.block.5.layer.0.SelfAttention.v.weight', 'decoder.block.5.layer.0.SelfAttention.o.weight', 'decoder.block.5.layer.0.layer_norm.weight', 'decoder.block.5.layer.1.EncDecAttention.q.weight', 'decoder.block.5.layer.1.EncDecAttention.k.weight', 'decoder.block.5.layer.1.EncDecAttention.v.weight', 'decoder.block.5.layer.1.EncDecAttention.o.weight', 'decoder.block.5.layer.1.layer_norm.weight', 'decoder.block.5.layer.2.DenseReluDense.wi.weight', 'decoder.block.5.layer.2.DenseReluDense.wo.weight', 'decoder.block.5.layer.2.layer_norm.weight', 'decoder.final_layer_norm.weight', 'lm_head.weight'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3xIpvzz-G8QN"
      },
      "source": [
        "# val\n",
        "phrases='pretty good food [SEP] VERY noisy wine list [SEP] Really delicious seasonal fare'\n",
        "text='''Very good   food indeed! Second time there. Love the food, pretty good wine list, but VERY noisy. So I guess it depends how much you want to shout or listen ... Worth trying, but with lots of energy and will to talk loud. Really delicious seasonal fare.'''\n",
        "# train\n",
        "phrases='Great customer </s></s> convenient parking </s></s> easy location </s></s> good service </s></s> really nice pharmacy tech </s></s> nicer location </s></s> never negative shopping experience </s></s> Great products </s></s> great prices </s></s> great store </s></s> super helpful guest </s></s> could orders </s></s> best experience </s></s> sweetest disposition </s></s> knowledgeable products'\n",
        "text='''If you are looking for something to do, take a break from walking the strip, or take a break from gambling, come watch a movie here! The theater is really old, a little difficult to locate, not visible from the strip (on the side of Denny's), and the theaters are super small, but hey it gives you something to do other than the usual things in Vegas. Prices are movies are definitely a bit pricey. Staff at the front seem to be friendly. Not much more to say about a movie theater. :) Enjoy.'''\n",
        "phrases='summarize: '+phrases.replace(' </s></s> ',',')\n",
        "encoded_phrases = tok.encode_plus(\n",
        "            phrases,\n",
        "            max_length = 128,\n",
        "            add_special_tokens = True,\n",
        "            return_attention_mask = True,\n",
        "            return_token_type_ids = True,\n",
        "            return_tensors = 'pt',\n",
        "            padding = 'max_length',\n",
        "            truncation = True,\n",
        "        )\n",
        "\n",
        "input_ids = encoded_phrases['input_ids'].to(device)\n",
        "att_mask = encoded_phrases['attention_mask'].to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XPDLUf_uHGSx",
        "outputId": "ff65dea4-a7f3-45f5-9983-21c3253b1b43"
      },
      "source": [
        "%%time\n",
        "output_greed = model.generate(\n",
        "    input_ids = input_ids, \n",
        "    attention_mask = att_mask,\n",
        "    min_length = 10,\n",
        "    max_length = 128,\n",
        "    early_stopping = True,\n",
        "    pad_token_id = tok.pad_token_id,\n",
        "    bos_token_id = tok.cls_token_id,\n",
        "    eos_token_id = tok.sep_token_id,\n",
        "    )\n",
        "\n",
        "greedy_sent = tok.decode(output_greed[0], skip_special_tokens=True)\n",
        "print(greedy_sent)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "This is a great store with great prices. Great products, great prices, and the pharmacy tech was really nice. The location is easy to get to, convenient parking, and the guest was super helpful. The owner was the sweetest disposition and made sure we had the best experience possible. I would recommend this to anyone looking for a nicer location. Great customer service.\n",
            "CPU times: user 23.4 s, sys: 26.8 ms, total: 23.4 s\n",
            "Wall time: 1.2 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MPCNt3S2HGWr",
        "outputId": "01024ea5-c01a-4378-c3d6-822c1f3bed33"
      },
      "source": [
        "%%time\n",
        "output_beam=model.generate(input_ids=input_ids,attention_mask=att_mask,min_length=10,max_length=128,\n",
        "                        early_stopping=True,pad_token_id=tok.pad_token_id,bos_token_id=tok.cls_token_id\n",
        "                        ,eos_token_id=tok.sep_token_id,num_beams=5)\n",
        "\n",
        "beam5_sent=tok.decode(output_beam[0], skip_special_tokens=True)\n",
        "print(beam5_sent)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "This is a great store. Great products, great prices, great prices. The pharmacy tech was really nice and had the sweetest disposition. The location is easy to get to and easy to get to and easy to get to. The guest at the front desk was super helpful with my orders and couldn't have been nicer. Great customer service and good service. This is the best experience I have ever had at a pharmacy.\n",
            "CPU times: user 49.4 s, sys: 271 ms, total: 49.7 s\n",
            "Wall time: 2.52 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f0-BzIItHGZZ",
        "outputId": "b0b78d6e-351b-4353-cebd-d0e829cc363a"
      },
      "source": [
        "%%time\n",
        "output_beam=model.generate(input_ids=input_ids,attention_mask=att_mask,min_length=10,max_length=128,\n",
        "                        early_stopping=True,pad_token_id=tok.pad_token_id,bos_token_id=tok.cls_token_id\n",
        "                        ,eos_token_id=tok.sep_token_id, num_beams=4)\n",
        "\n",
        "beamn_sent=tok.decode(output_beam[0], skip_special_tokens=True)\n",
        "print(beamn_sent)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "This is a great store! Great products, great prices, great prices. The pharmacy tech was really nice and had the sweetest disposition. The location is easy to get to and easy to get to and easy to get to. The guest at the front desk was super helpful with my orders and couldn't have asked for a better experience. I will definitely be back!\n",
            "CPU times: user 48.6 s, sys: 476 ms, total: 49.1 s\n",
            "Wall time: 2.48 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sfg65qo0HGcY",
        "outputId": "2660d046-7004-487f-83fc-b161f6a7f306"
      },
      "source": [
        "%%time\n",
        "evaluateSummary([greedy_sent],[text]),evaluateSummary([beam5_sent],[text]),evaluateSummary([beamn_sent],[text])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 8.18 ms, sys: 0 ns, total: 8.18 ms\n",
            "Wall time: 8.13 ms\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'Rouge-1': 0.22929936305732482,\n",
              "  'Rouge-2': 0.025806451612903226,\n",
              "  'Rouge-L': 0.15286624203821655},\n",
              " {'Rouge-1': 0.23668639053254437,\n",
              "  'Rouge-2': 0.02395209580838324,\n",
              "  'Rouge-L': 0.14201183431952663},\n",
              " {'Rouge-1': 0.2767295597484277,\n",
              "  'Rouge-2': 0.025477707006369428,\n",
              "  'Rouge-L': 0.1509433962264151})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "scpyeLPuAdAe"
      },
      "source": [
        "def eval_rogue(model, dev, data):\n",
        "    orig, greedy, beam=[],[],[]\n",
        "    model.to(dev)\n",
        "    model.eval()\n",
        "\n",
        "    test_dataset = dataset(data)\n",
        "\n",
        "    test_loader = torch.utils.data.DataLoader(\n",
        "        test_dataset,\n",
        "        batch_size = 32,\n",
        "    )\n",
        "\n",
        "    # def batch(iterable, n=1):\n",
        "    #     l = len(iterable)\n",
        "    #     for ndx in range(0, l, n):\n",
        "    #         yield iterable[ndx:min(ndx + n, l)]\n",
        "\n",
        "    for row in tqdm(test_loader): \n",
        "        orig += row['review']\n",
        "        input_ids = row['p_input_ids'].to(dev)\n",
        "        att_mask = row['p_attention_mask'].to(dev)\n",
        "\n",
        "        output_greed = model.generate(\n",
        "            input_ids=input_ids,\n",
        "            attention_mask=att_mask,\n",
        "            min_length=10,\n",
        "            max_length=50,\n",
        "            early_stopping=True,\n",
        "            pad_token_id=tok.pad_token_id,\n",
        "            bos_token_id=tok.cls_token_id,\n",
        "            eos_token_id=tok.sep_token_id\n",
        "            )\n",
        "\n",
        "        greedy += [tok.decode(output, skip_special_tokens=True) for output in output_greed]\n",
        "\n",
        "        output_beam = model.generate(\n",
        "            input_ids=input_ids,\n",
        "            attention_mask=att_mask,\n",
        "            min_length=10,\n",
        "            max_length=50,\n",
        "            early_stopping=True,\n",
        "            pad_token_id=tok.pad_token_id,\n",
        "            bos_token_id=tok.cls_token_id,\n",
        "            eos_token_id=tok.sep_token_id,\n",
        "            num_beams=4)\n",
        "\n",
        "        beam += [tok.decode(output, skip_special_tokens=True) for output in output_beam]\n",
        "\n",
        "    data['greedy'] = greedy\n",
        "    data['beam'] = beam    \n",
        "    print(len(orig), len(beam))    \n",
        "    \n",
        "    return {'greedy':evaluateSummary(greedy, orig), 'beam':evaluateSummary(beam, orig)}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w9eONBrbA4VB",
        "outputId": "7a810b43-543f-4281-d4aa-5c6e3e7da609"
      },
      "source": [
        "test_set = pd.read_csv(\"/content/data/yelp-default/test_gold_8_15_all_all_300_8.csv\")\n",
        "test_set.columns=['eid', 'rids', 'n', 'text', 'review_0', 'review_1', 'review_2',\n",
        "       'review_3', 'review_4', 'review_5', 'review_6', 'review_7',\n",
        "       'extraction', 'phrases']\n",
        "       \n",
        "scores = eval_rogue(model, device, test_set)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 7/7 [01:36<00:00, 13.78s/it]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "200 200\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "inkqT1o_JKu-",
        "outputId": "a1bee87b-420e-4b03-d94b-086880f61157"
      },
      "source": [
        "scores\n",
        "# {'beam': {'Rouge-1': 0.23833223304206289,\n",
        "#   'Rouge-2': 0.03479748240427835,\n",
        "#   'Rouge-L': 0.15303731099861098},\n",
        "#  'greedy': {'Rouge-1': 0.240540043067736,\n",
        "#   'Rouge-2': 0.03369968209532295,\n",
        "#   'Rouge-L': 0.15454299725336362}}\n",
        "\n",
        "# {'beam': {'Rouge-1': 0.2392596713943409,\n",
        "#   'Rouge-2': 0.035799747079077696,\n",
        "#   'Rouge-L': 0.15874841295479636},\n",
        "#  'greedy': {'Rouge-1': 0.2369498751709391,\n",
        "#   'Rouge-2': 0.03521518570567792,\n",
        "#   'Rouge-L': 0.1629997963483346}}"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'beam': {'Rouge-1': 0.23731161836100853,\n",
              "  'Rouge-2': 0.033559542929670234,\n",
              "  'Rouge-L': 0.1559495411401135},\n",
              " 'greedy': {'Rouge-1': 0.23818624673866842,\n",
              "  'Rouge-2': 0.03621578068887971,\n",
              "  'Rouge-L': 0.1577551971461499}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SV9XfJL6nGBz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "e57a8632-3887-4491-c6d8-38c62427ed0d"
      },
      "source": [
        "test_set"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>eid</th>\n",
              "      <th>rids</th>\n",
              "      <th>n</th>\n",
              "      <th>text</th>\n",
              "      <th>review_0</th>\n",
              "      <th>review_1</th>\n",
              "      <th>review_2</th>\n",
              "      <th>review_3</th>\n",
              "      <th>review_4</th>\n",
              "      <th>review_5</th>\n",
              "      <th>review_6</th>\n",
              "      <th>review_7</th>\n",
              "      <th>extraction</th>\n",
              "      <th>phrases</th>\n",
              "      <th>greedy</th>\n",
              "      <th>beam</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>fjufqwFSQrUhLqoYGTklHQ</td>\n",
              "      <td>1,5,0,3,4,7,2,6</td>\n",
              "      <td>8</td>\n",
              "      <td>Fresh food, high quality food, delicious and M...</td>\n",
              "      <td>I tried to order steak kebob but they made bee...</td>\n",
              "      <td>Very delicious food in love with cucumber drin...</td>\n",
              "      <td>Woww! My order:   Chicken Schwarma with a side...</td>\n",
              "      <td>I was thinking this would be more of a sit dow...</td>\n",
              "      <td>Parsley Modern Mediterranean is wonderful. Ver...</td>\n",
              "      <td>The food always taste fresh and leaves me very...</td>\n",
              "      <td>Now this place is really good i always drive p...</td>\n",
              "      <td>This is Chipotle for Mediterranean food. And i...</td>\n",
              "      <td>good,Price,value-for-money,positive;okay,Taste...</td>\n",
              "      <td>good Price [SEP] okay Taste [SEP] Very delicio...</td>\n",
              "      <td>This is my favorite Mediterranean place in the...</td>\n",
              "      <td>This place is really good. Very delicious. Ver...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2JsLzYF8rUalwpm5LDEcog</td>\n",
              "      <td>9,12,8,14,13,10,15,11</td>\n",
              "      <td>8</td>\n",
              "      <td>The food is great here. It can be a little exp...</td>\n",
              "      <td>Food very good. Small unassuming atmosphere wi...</td>\n",
              "      <td>Fantastic place with phenomenal food. Very uni...</td>\n",
              "      <td>just excellent. friendly staff, good food, fun...</td>\n",
              "      <td>This is my second time visiting --- and the fo...</td>\n",
              "      <td>Very cool decor and atmosphere, but extremely ...</td>\n",
              "      <td>Over priced specials. Not good. The clam chowd...</td>\n",
              "      <td>Very good! Many items made from scratch and fi...</td>\n",
              "      <td>The Falls' hidden gem--kooky decor, decadent f...</td>\n",
              "      <td>good,food,food -&gt; quality,positive;excellent,s...</td>\n",
              "      <td>good food [SEP] excellent seafood [SEP] unassu...</td>\n",
              "      <td>Fantastic place! Family atmosphere, friendly s...</td>\n",
              "      <td>Fantastic place. Family atmosphere, friendly s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>6C_8Mh4lmLc_QEs3hHleBg</td>\n",
              "      <td>18,22,16,23,21,19,17,20</td>\n",
              "      <td>8</td>\n",
              "      <td>Mexican food that reminds me of what my mom us...</td>\n",
              "      <td>We call this \" Red Chicken. It's consistently ...</td>\n",
              "      <td>I like this place. The food here is like what ...</td>\n",
              "      <td>It could be a lot better, but with no competit...</td>\n",
              "      <td>One of the best deals and delicious beef and c...</td>\n",
              "      <td>This is as close as the chicken that you will ...</td>\n",
              "      <td>My bro and I have found this place by chance. ...</td>\n",
              "      <td>Nice simple Mexican food at a great price, lov...</td>\n",
              "      <td>To my homesick Indianz from Arizona, if you wa...</td>\n",
              "      <td>basic charbroiled,chicken,food -&gt; quality,nega...</td>\n",
              "      <td>basic charbroiled chicken [SEP] love chicken [...</td>\n",
              "      <td>Nice simple Mexican food at a great price. The...</td>\n",
              "      <td>Nice simple Mexican food at a great price. I l...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5VXsbrqyJx0a4iaa43RNFA</td>\n",
              "      <td>26,24,25,29,27,30,28,31</td>\n",
              "      <td>8</td>\n",
              "      <td>This is an ok breakfast joint.  The food is ki...</td>\n",
              "      <td>Great hip breakfast in Gilbert. Food and waitr...</td>\n",
              "      <td>I think Over Easy is a very nice breakfast pla...</td>\n",
              "      <td>Not good at all. The young staff basically ign...</td>\n",
              "      <td>We love the service and food. The staff is alw...</td>\n",
              "      <td>Visited for the first time this morning and ev...</td>\n",
              "      <td>Had the biscuits and gravy...they were OK. ( B...</td>\n",
              "      <td>I want to give a 2 stars because the service s...</td>\n",
              "      <td>I really never post any reviews but this place...</td>\n",
              "      <td>very nice,breakfast place,food -&gt; quality,posi...</td>\n",
              "      <td>very nice breakfast place [SEP] great food [SE...</td>\n",
              "      <td>Great hip breakfast. I love the service and th...</td>\n",
              "      <td>This is a very nice breakfast place. The staff...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5iHctUjkQTGwEvOaBkwMRQ</td>\n",
              "      <td>38,36,34,39,32,35,33,37</td>\n",
              "      <td>8</td>\n",
              "      <td>This place has a nice buffet, which in itself ...</td>\n",
              "      <td>Awesome place for nice coffee and great delici...</td>\n",
              "      <td>This place is my new favorite! Their coffee is...</td>\n",
              "      <td>I was looking for just a place to get breakfas...</td>\n",
              "      <td>Great lunch spot downtown. Food tastes good. T...</td>\n",
              "      <td>I ate here this week and it was good place for...</td>\n",
              "      <td>We found this place walking toward another bre...</td>\n",
              "      <td>This review is for the salad bar, I feel like ...</td>\n",
              "      <td>Let`s start with the good stuff first. Clean p...</td>\n",
              "      <td>favorite,restaurant for lunch,food -&gt; quality,...</td>\n",
              "      <td>favorite restaurant for lunch [SEP] soo good c...</td>\n",
              "      <td>This is my favorite restaurant for lunch. The ...</td>\n",
              "      <td>This is my favorite restaurant for lunch. The ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>195</th>\n",
              "      <td>4iACB0ppsvwA2mFvLJoNbA</td>\n",
              "      <td>1563,1566,1562,1565,1564,1561,1560,1567</td>\n",
              "      <td>8</td>\n",
              "      <td>This place is a pretty sweet tattoo parlor!  i...</td>\n",
              "      <td>Brandon did a great job on my first tattoo. Wa...</td>\n",
              "      <td>Yooooo this place is dope!! We got there an ho...</td>\n",
              "      <td>From Heidi: Justin is the bomb, he provided ex...</td>\n",
              "      <td>Kevin did a amazing job on the cover up i need...</td>\n",
              "      <td>Thank you Nick Giordono for the sick tattoo! I...</td>\n",
              "      <td>came here on a Saturday after calling to inqui...</td>\n",
              "      <td>Went in for a small tattoo on my wrist and had...</td>\n",
              "      <td>My twin sister and I just got our first tattoo...</td>\n",
              "      <td>good,experience overall,restaurant -&gt; atmosphe...</td>\n",
              "      <td>good experience overall [SEP] dope place [SEP]...</td>\n",
              "      <td>This place is dope! Terry is a fantastic busin...</td>\n",
              "      <td>This place is dope. Terry is a fantastic busin...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>196</th>\n",
              "      <td>0uwQIGlKICIYuRtayVnh7g</td>\n",
              "      <td>1573,1571,1574,1570,1572,1569,1568,1575</td>\n",
              "      <td>8</td>\n",
              "      <td>This place is kind of weird. The food menu is ...</td>\n",
              "      <td>Pointless to order on line,   staff unapologet...</td>\n",
              "      <td>I've come here a few times before and never ha...</td>\n",
              "      <td>Was curious about this place. Pasta was hot an...</td>\n",
              "      <td>If chipotle had a stupid Italian cousin it wou...</td>\n",
              "      <td>Disappointed. Plus, we were waiting in line an...</td>\n",
              "      <td>This is the worst piada i have been to. They w...</td>\n",
              "      <td>Nice place for good food when you're in a hurr...</td>\n",
              "      <td>I used to love Piada, but this particular loca...</td>\n",
              "      <td>rude,staff,staff,negative;out of several ingre...</td>\n",
              "      <td>rude staff [SEP] out of several ingredients st...</td>\n",
              "      <td>Nice place, rude staff, long lines, awkward fo...</td>\n",
              "      <td>Nice place, rude staff, long lines, awkward fo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>197</th>\n",
              "      <td>4yAvytbVKHqSYAo3mkI9OA</td>\n",
              "      <td>1580,1583,1576,1582,1577,1578,1579,1581</td>\n",
              "      <td>8</td>\n",
              "      <td>This place has a excellent price point, and th...</td>\n",
              "      <td>I came accidentally to this place and guess wh...</td>\n",
              "      <td>Good food. Nice people. The beef tikka is defi...</td>\n",
              "      <td>Food is very good, service and atmosphere real...</td>\n",
              "      <td>Food is ok and definitely not PERSIAN. If you ...</td>\n",
              "      <td>Best kabob in town, I wish I could give more s...</td>\n",
              "      <td>Perfect in every way! The spices, the tenderne...</td>\n",
              "      <td>Yum! Hubs brought home \" to go \" order. Super ...</td>\n",
              "      <td>Wow ...... This place is the bomb. Have been h...</td>\n",
              "      <td>very good,Food,food -&gt; quality,positive;best,l...</td>\n",
              "      <td>very good Food [SEP] best lunches [SEP] delici...</td>\n",
              "      <td>I really appreciate this restaurant. Nice peop...</td>\n",
              "      <td>One of the best lunches I've had in a long tim...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>198</th>\n",
              "      <td>4c19YWOjPmbFUK4-V2GEvg</td>\n",
              "      <td>1584,1588,1587,1585,1590,1586,1591,1589</td>\n",
              "      <td>8</td>\n",
              "      <td>This place is just \"meh.\"  I wish the menu had...</td>\n",
              "      <td>I've only eaten appetizers here but they are a...</td>\n",
              "      <td>I really love the idea of this place (menus ta...</td>\n",
              "      <td>As a teacher you would think I would love this...</td>\n",
              "      <td>Went here yesterday, my girlfriend ordered a b...</td>\n",
              "      <td>It's \" i eat \" I guess. Food is nothing to WRI...</td>\n",
              "      <td>Food is decent, and the drinks are ok. Not a h...</td>\n",
              "      <td>Trivia on tuesday was fun, and the half off ap...</td>\n",
              "      <td>The bald bartender is the worst I've ever seen...</td>\n",
              "      <td>Huge,portions-,food-quantity,positive;good-,dr...</td>\n",
              "      <td>Huge portions- [SEP] good- drinks [SEP] very c...</td>\n",
              "      <td>This is a very cool option for pub food. Huge ...</td>\n",
              "      <td>This is a very cool option for pub food. Huge ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>199</th>\n",
              "      <td>1_Dg-GguWatioW4mvFQKQA</td>\n",
              "      <td>1593,1592,1598,1594,1599,1597,1595,1596</td>\n",
              "      <td>8</td>\n",
              "      <td>They have a long list of meals but the sushi i...</td>\n",
              "      <td>The sushis were nice. I especially liked the D...</td>\n",
              "      <td>Great sushi and very friendly staff (even the ...</td>\n",
              "      <td>Sushi with great mouthfeel, well - executed ri...</td>\n",
              "      <td>We went on a surprise date to a new place. The...</td>\n",
              "      <td>Wonderful sushi ver much more on the tradition...</td>\n",
              "      <td>We really like this place--super friendly owne...</td>\n",
              "      <td>I heard great things about this place, which i...</td>\n",
              "      <td>Stunningly bad ... Not sure what else to say b...</td>\n",
              "      <td>Wonderful,sushi,food -&gt; quality,positive;nice,...</td>\n",
              "      <td>Wonderful sushi [SEP] nice sushis [SEP] liked ...</td>\n",
              "      <td>Wonderful sushi and very friendly staff. Love ...</td>\n",
              "      <td>Wonderful sushi and very friendly staff. Love ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>200 rows × 16 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                        eid  ...                                               beam\n",
              "0    fjufqwFSQrUhLqoYGTklHQ  ...  This place is really good. Very delicious. Ver...\n",
              "1    2JsLzYF8rUalwpm5LDEcog  ...  Fantastic place. Family atmosphere, friendly s...\n",
              "2    6C_8Mh4lmLc_QEs3hHleBg  ...  Nice simple Mexican food at a great price. I l...\n",
              "3    5VXsbrqyJx0a4iaa43RNFA  ...  This is a very nice breakfast place. The staff...\n",
              "4    5iHctUjkQTGwEvOaBkwMRQ  ...  This is my favorite restaurant for lunch. The ...\n",
              "..                      ...  ...                                                ...\n",
              "195  4iACB0ppsvwA2mFvLJoNbA  ...  This place is dope. Terry is a fantastic busin...\n",
              "196  0uwQIGlKICIYuRtayVnh7g  ...  Nice place, rude staff, long lines, awkward fo...\n",
              "197  4yAvytbVKHqSYAo3mkI9OA  ...  One of the best lunches I've had in a long tim...\n",
              "198  4c19YWOjPmbFUK4-V2GEvg  ...  This is a very cool option for pub food. Huge ...\n",
              "199  1_Dg-GguWatioW4mvFQKQA  ...  Wonderful sushi and very friendly staff. Love ...\n",
              "\n",
              "[200 rows x 16 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5jzXw_iEl8Sx",
        "outputId": "1ecd330c-899e-4247-a7e8-bc4c749a90d2"
      },
      "source": [
        "!zip op_Roberta60k_10.zip my_model"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  adding: my_model (deflated 11%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P7GvkTP-lF5p"
      },
      "source": [
        "# Import PyDrive and associated libraries.\n",
        "# This only needs to be done once in a notebook.\n",
        "# !pip install httplib2==0.15.0 -q\n",
        "# !pip install pydrive --upgrade -q\n",
        "# !pip install google-api-python-client==1.6 -q\n",
        "\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "# Authenticate and create the PyDrive client.\n",
        "# This only needs to be done once in a notebook.\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CB1UJee4lhvB"
      },
      "source": [
        "# Create & upload a text file.\n",
        "uploaded = drive.CreateFile()\n",
        "uploaded.SetContentFile('op_Roberta60k_10.zip')\n",
        "uploaded.Upload()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7sk_ySN_FS4A"
      },
      "source": [
        "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
        "tokenizer = T5Tokenizer.from_pretrained('t5-base')\n",
        "model = T5ForConditionalGeneration.from_pretrained('t5-base', return_dict=True)\n",
        "input_ids = tokenizer('The <extra_id_0> walks in <extra_id_1> park', return_tensors='pt').input_ids\n",
        "labels = tokenizer('<extra_id_0> cute dog <extra_id_1> the <extra_id_2>', return_tensors='pt').input_ids\n",
        "outputs = model(input_ids=input_ids, labels=labels)\n",
        "loss = outputs.loss\n",
        "logits = outputs.logits\n",
        "input_ids = tokenizer(\"summarize: studies have shown that owning a dog is good for you \", return_tensors=\"pt\").input_ids  # Batch size 1\n",
        "outputs = model.generate(input_ids)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "UITnIFgDNO2G",
        "outputId": "68926e3b-5eae-4180-c487-d0a3b6a5af04"
      },
      "source": [
        "tokenizer.decode(outputs[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'owning a dog is good for you, according to studies. a dog is'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-QS-dgtlagwE"
      },
      "source": [
        "encoded_text = tokenizer.encode_plus(\n",
        "            train.iloc[0]['phrases'],\n",
        "            truncation=True,\n",
        "            max_length=128,\n",
        "            add_special_tokens=True,\n",
        "            padding=True,\n",
        "            return_attention_mask=True,\n",
        "            return_token_type_ids=True,\n",
        "            return_tensors='pt',\n",
        "        )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "nHiBoZhsagtr",
        "outputId": "46b1febe-a266-47ad-f7d7-3b61136c6112"
      },
      "source": [
        "tokenizer.decode(encoded_text.input_ids[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'really old theater [SEP] super small theaters [SEP] bit pricey movies [SEP] friendly Staff'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tX7g-FaKgeIG",
        "outputId": "e4b6cabf-b1ed-4938-c45c-f8630021486e"
      },
      "source": [
        "dict(outputs).keys()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['last_hidden_state', 'past_key_values', 'encoder_last_hidden_state'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "NNj2-XiCkEy8",
        "outputId": "45481940-2ec3-48b2-bd2b-28ca803d862b"
      },
      "source": [
        "train.iloc[0]['phrases']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'really old theater [SEP] super small theaters [SEP] bit pricey movies [SEP] friendly Staff'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vsIagGHcJu5m"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}