FastTextComparer/Fast_Transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Fast_Transformers.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyM1ew6tNE9aPMZOL4S5PctT"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "fd48148492e5425eb3423cff11c958a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_be64cda0c22b4d6e8e8f04928ae8bc09",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_552658a0395a4cf5917aa49ab5e9fac1",
              "IPY_MODEL_f52b4fde0fbd495588b5b531e0a3dfe0",
              "IPY_MODEL_8b837b1cdacf41eaac86f28da50dae4e"
            ]
          }
        },
        "be64cda0c22b4d6e8e8f04928ae8bc09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "552658a0395a4cf5917aa49ab5e9fac1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4e0024c5258448c68544bc0b9ef1d3db",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d2fee207231d493f90a88127f3b4dd5a"
          }
        },
        "f52b4fde0fbd495588b5b531e0a3dfe0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e7343c1ec1ce4aa5ad546a8876396f00",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e130f7761bc54214bc71e9fda769a8f1"
          }
        },
        "8b837b1cdacf41eaac86f28da50dae4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2d6e4e45285e4a8aa4e512f9589600ca",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 874kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d148aaf554f34ede97cc7b3dea0a9771"
          }
        },
        "4e0024c5258448c68544bc0b9ef1d3db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d2fee207231d493f90a88127f3b4dd5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e7343c1ec1ce4aa5ad546a8876396f00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e130f7761bc54214bc71e9fda769a8f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d6e4e45285e4a8aa4e512f9589600ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d148aaf554f34ede97cc7b3dea0a9771": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "036775b9c2aa4ff08ed7b11452520bcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0e4324b1def0441695bb3333dca333ec",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f9a847abb0954901ae8e9ce57989e249",
              "IPY_MODEL_5ea4a8b984e94711aafbccb7a6e96db3",
              "IPY_MODEL_7645cabe77fd46d5b9558fc81fd3a54d"
            ]
          }
        },
        "0e4324b1def0441695bb3333dca333ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f9a847abb0954901ae8e9ce57989e249": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0ccb17af402840d79eaf4b58f38d0a70",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_02ca55e26c844b16be697e27dbe8cb7d"
          }
        },
        "5ea4a8b984e94711aafbccb7a6e96db3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_36297d9d00e74676aa0654f8874c8a66",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2027dc84686649cf9e30db019d892c5c"
          }
        },
        "7645cabe77fd46d5b9558fc81fd3a54d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_384009cbf64e4d9dac730cb77c1a18cc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 1.19MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2a8d4b74abc2486a9d4e75898e61cdb3"
          }
        },
        "0ccb17af402840d79eaf4b58f38d0a70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "02ca55e26c844b16be697e27dbe8cb7d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "36297d9d00e74676aa0654f8874c8a66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2027dc84686649cf9e30db019d892c5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "384009cbf64e4d9dac730cb77c1a18cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2a8d4b74abc2486a9d4e75898e61cdb3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4aecd87af0d84086821df6763aeef9f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f246c5e1c5fb4eba91e2363ab9fd2bc6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fca9707a7a0941fdbd2c99470449b53d",
              "IPY_MODEL_e9f7973aa1f34b448007d07b1cb8733f",
              "IPY_MODEL_d8b64600a93e406492e2d2e48baa1b3e"
            ]
          }
        },
        "f246c5e1c5fb4eba91e2363ab9fd2bc6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fca9707a7a0941fdbd2c99470449b53d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2faa981753e34a6aa05ff946af9f7491",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_558a4ad7604f40fb8bcd0736a32f7a59"
          }
        },
        "e9f7973aa1f34b448007d07b1cb8733f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c5a1b9d9447f45418771eca06cd40c61",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b584da70945e4422baee14d7f4dffaf8"
          }
        },
        "d8b64600a93e406492e2d2e48baa1b3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0fbc7c38b03d4704ab1424a4311abea7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 615B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c5f8017680f84bd89dbea78888b26b4d"
          }
        },
        "2faa981753e34a6aa05ff946af9f7491": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "558a4ad7604f40fb8bcd0736a32f7a59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c5a1b9d9447f45418771eca06cd40c61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b584da70945e4422baee14d7f4dffaf8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0fbc7c38b03d4704ab1424a4311abea7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c5f8017680f84bd89dbea78888b26b4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d3e8c9c82214f3eb3ceaf1d8514b39a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7657c6bfbe2c4918996f8285d7096a13",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ba1609d5f7ad431fad0f0a0deeb39713",
              "IPY_MODEL_0cbbcbe820824b98a78beb45b0bc5753",
              "IPY_MODEL_b3bd6932fd2145b6a55506d919c0c71e"
            ]
          }
        },
        "7657c6bfbe2c4918996f8285d7096a13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ba1609d5f7ad431fad0f0a0deeb39713": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c88592959b234738985c79824de617c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_46d0a0005f034cdf9b33132b7035d064"
          }
        },
        "0cbbcbe820824b98a78beb45b0bc5753": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6300ac198579418da8be8b84b8c2db59",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 570,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 570,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c49bd70e069e4b48974c8c78841c63c7"
          }
        },
        "b3bd6932fd2145b6a55506d919c0c71e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fc1be6853f4b4cd2a794b534d151c2c1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 570/570 [00:00&lt;00:00, 11.4kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_359e9493b99746e4bb4618b2e93efcd6"
          }
        },
        "c88592959b234738985c79824de617c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "46d0a0005f034cdf9b33132b7035d064": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6300ac198579418da8be8b84b8c2db59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c49bd70e069e4b48974c8c78841c63c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fc1be6853f4b4cd2a794b534d151c2c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "359e9493b99746e4bb4618b2e93efcd6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3ae14a8c1e914d389f8bd1552ad7b579": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_142011f77ae1471fb0151f75a731982e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1944de346df5416a88a89c935fc2dfd8",
              "IPY_MODEL_17fad2300f55468b91c3cbecf781789e",
              "IPY_MODEL_d00d653b9f424a7d870d0c3456108a9f"
            ]
          }
        },
        "142011f77ae1471fb0151f75a731982e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1944de346df5416a88a89c935fc2dfd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_67b13cdc7a2c47dfaa3cbbf8a38b3b22",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e9e375b4f51e4609b68780946aaefbc4"
          }
        },
        "17fad2300f55468b91c3cbecf781789e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_24a43fdd44504226a46383a0fa66194b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2897d1d5ec9e4378b38ea6a449d51f75"
          }
        },
        "d00d653b9f424a7d870d0c3456108a9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1331205413864c22a614326c36c961c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:13&lt;00:00, 33.1MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5603eb7def4f4958b138e0b46d8bc57a"
          }
        },
        "67b13cdc7a2c47dfaa3cbbf8a38b3b22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e9e375b4f51e4609b68780946aaefbc4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "24a43fdd44504226a46383a0fa66194b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2897d1d5ec9e4378b38ea6a449d51f75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1331205413864c22a614326c36c961c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5603eb7def4f4958b138e0b46d8bc57a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pqSeNAlV99cM",
        "outputId": "aadb8c24-f13a-4b19-9847-f07a00b31488"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.10.0-py3-none-any.whl (2.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.8 MB 5.3 MB/s \n",
            "\u001b[?25hCollecting pyyaml>=5.1\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 34.2 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 32.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 59.0 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub>=0.0.12\n",
            "  Downloading huggingface_hub-0.0.16-py3-none-any.whl (50 kB)\n",
            "\u001b[K     |████████████████████████████████| 50 kB 6.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.4)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub>=0.0.12->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.5.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Installing collected packages: tokenizers, sacremoses, pyyaml, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.0.16 pyyaml-5.4.1 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.10.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wRpt9v7U99e5"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report\n",
        "import transformers\n",
        "from transformers import AutoModel, BertTokenizerFast\n",
        "\n",
        "# specify GPU\n",
        "#device = torch.device(\"cuda\")"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_AWWkT4-99hQ"
      },
      "source": [
        "file=\"https://raw.githubusercontent.com/prateekjoshi565/Fine-Tuning-BERT/master/spamdata_v2.csv\""
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "2tUvyO1_99j6",
        "outputId": "fa9ca06d-28ec-4ec7-d06f-d89de5dae20e"
      },
      "source": [
        "df = pd.read_csv(file)\n",
        "df.head()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>Ok lar... Joking wif u oni...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   label                                               text\n",
              "0      0  Go until jurong point, crazy.. Available only ...\n",
              "1      0                      Ok lar... Joking wif u oni...\n",
              "2      1  Free entry in 2 a wkly comp to win FA Cup fina...\n",
              "3      0  U dun say so early hor... U c already then say...\n",
              "4      0  Nah I don't think he goes to usf, he lives aro..."
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5nHkOhaP99mQ"
      },
      "source": [
        "# split train dataset into train, validation and test sets\n",
        "train_text, temp_text, train_labels, temp_labels = train_test_split(df['text'], df['label'], \n",
        "                                                                    random_state=2018, \n",
        "                                                                    test_size=0.3, \n",
        "                                                                    stratify=df['label'])\n",
        "\n",
        "\n",
        "val_text, test_text, val_labels, test_labels = train_test_split(temp_text, temp_labels, \n",
        "                                                                random_state=2018, \n",
        "                                                                test_size=0.5, \n",
        "                                                                stratify=temp_labels)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CE_4Z8-v99o0",
        "outputId": "74510186-0d4d-41c8-9c01-cfa291463dbe"
      },
      "source": [
        "!pip install fast-transformer-pytorch"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting fast-transformer-pytorch\n",
            "  Downloading fast_transformer_pytorch-0.0.4-py3-none-any.whl (4.6 kB)\n",
            "Collecting rotary-embedding-torch\n",
            "  Downloading rotary_embedding_torch-0.1.0-py3-none-any.whl (4.1 kB)\n",
            "Requirement already satisfied: torch>=1.6 in /usr/local/lib/python3.7/dist-packages (from fast-transformer-pytorch) (1.9.0+cu102)\n",
            "Collecting einops>=0.3\n",
            "  Downloading einops-0.3.2-py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.6->fast-transformer-pytorch) (3.7.4.3)\n",
            "Installing collected packages: einops, rotary-embedding-torch, fast-transformer-pytorch\n",
            "Successfully installed einops-0.3.2 fast-transformer-pytorch-0.0.4 rotary-embedding-torch-0.1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "fd48148492e5425eb3423cff11c958a3",
            "be64cda0c22b4d6e8e8f04928ae8bc09",
            "552658a0395a4cf5917aa49ab5e9fac1",
            "f52b4fde0fbd495588b5b531e0a3dfe0",
            "8b837b1cdacf41eaac86f28da50dae4e",
            "4e0024c5258448c68544bc0b9ef1d3db",
            "d2fee207231d493f90a88127f3b4dd5a",
            "e7343c1ec1ce4aa5ad546a8876396f00",
            "e130f7761bc54214bc71e9fda769a8f1",
            "2d6e4e45285e4a8aa4e512f9589600ca",
            "d148aaf554f34ede97cc7b3dea0a9771",
            "036775b9c2aa4ff08ed7b11452520bcf",
            "0e4324b1def0441695bb3333dca333ec",
            "f9a847abb0954901ae8e9ce57989e249",
            "5ea4a8b984e94711aafbccb7a6e96db3",
            "7645cabe77fd46d5b9558fc81fd3a54d",
            "0ccb17af402840d79eaf4b58f38d0a70",
            "02ca55e26c844b16be697e27dbe8cb7d",
            "36297d9d00e74676aa0654f8874c8a66",
            "2027dc84686649cf9e30db019d892c5c",
            "384009cbf64e4d9dac730cb77c1a18cc",
            "2a8d4b74abc2486a9d4e75898e61cdb3",
            "4aecd87af0d84086821df6763aeef9f8",
            "f246c5e1c5fb4eba91e2363ab9fd2bc6",
            "fca9707a7a0941fdbd2c99470449b53d",
            "e9f7973aa1f34b448007d07b1cb8733f",
            "d8b64600a93e406492e2d2e48baa1b3e",
            "2faa981753e34a6aa05ff946af9f7491",
            "558a4ad7604f40fb8bcd0736a32f7a59",
            "c5a1b9d9447f45418771eca06cd40c61",
            "b584da70945e4422baee14d7f4dffaf8",
            "0fbc7c38b03d4704ab1424a4311abea7",
            "c5f8017680f84bd89dbea78888b26b4d",
            "2d3e8c9c82214f3eb3ceaf1d8514b39a",
            "7657c6bfbe2c4918996f8285d7096a13",
            "ba1609d5f7ad431fad0f0a0deeb39713",
            "0cbbcbe820824b98a78beb45b0bc5753",
            "b3bd6932fd2145b6a55506d919c0c71e",
            "c88592959b234738985c79824de617c9",
            "46d0a0005f034cdf9b33132b7035d064",
            "6300ac198579418da8be8b84b8c2db59",
            "c49bd70e069e4b48974c8c78841c63c7",
            "fc1be6853f4b4cd2a794b534d151c2c1",
            "359e9493b99746e4bb4618b2e93efcd6"
          ]
        },
        "id": "s43a5Pr--g30",
        "outputId": "195f1850-f08c-4bb6-a111-e7c4678dbd91"
      },
      "source": [
        "from fast_transformer_pytorch import FastTransformer\n",
        "\n",
        "# Load the BERT tokenizer\n",
        "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fd48148492e5425eb3423cff11c958a3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "036775b9c2aa4ff08ed7b11452520bcf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4aecd87af0d84086821df6763aeef9f8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2d3e8c9c82214f3eb3ceaf1d8514b39a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121,
          "referenced_widgets": [
            "3ae14a8c1e914d389f8bd1552ad7b579",
            "142011f77ae1471fb0151f75a731982e",
            "1944de346df5416a88a89c935fc2dfd8",
            "17fad2300f55468b91c3cbecf781789e",
            "d00d653b9f424a7d870d0c3456108a9f",
            "67b13cdc7a2c47dfaa3cbbf8a38b3b22",
            "e9e375b4f51e4609b68780946aaefbc4",
            "24a43fdd44504226a46383a0fa66194b",
            "2897d1d5ec9e4378b38ea6a449d51f75",
            "1331205413864c22a614326c36c961c9",
            "5603eb7def4f4958b138e0b46d8bc57a"
          ]
        },
        "id": "5zGgxzL8_Auk",
        "outputId": "9938fd8e-3fbf-4243-fd74-edbd27cbb38e"
      },
      "source": [
        "# import BERT-base pretrained model\n",
        "bert = AutoModel.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3ae14a8c1e914d389f8bd1552ad7b579",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RIBarCAj_DjE",
        "outputId": "af36b7df-16de-4a21-a13a-405b3e82f165"
      },
      "source": [
        "# sample data\n",
        "text = [\"this is a bert model tutorial\", \"we will fine-tune a bert model\"]\n",
        "\n",
        "# encode text\n",
        "sent_id = tokenizer.batch_encode_plus(text, padding=True)\n",
        "\n",
        "# output\n",
        "print(sent_id)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [[101, 2023, 2003, 1037, 14324, 2944, 14924, 4818, 102, 0], [101, 2057, 2097, 2986, 1011, 8694, 1037, 14324, 2944, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "Ib3bK7r-_Dlh",
        "outputId": "d7d1d82b-8cd5-4075-ebf8-ae80d67353c8"
      },
      "source": [
        "# get length of all the messages in the train set\n",
        "seq_len = [len(i.split()) for i in train_text]\n",
        "\n",
        "pd.Series(seq_len).hist(bins = 30)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7fcadb6babd0>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUhklEQVR4nO3df5Dcd13H8efbxhboYdIfzE0niV7QiFMbleamrYMyd8aBNEVSFZl2OpBgnYxji8XWoUFG66jMBBURRsSJpkPQyhURprEtQgw9Gf5IpamlSVtKryVIbkIqtASPVjH69o/9nG7Pu9z+SHZv83k+Zm7uu5/vZ3df++32td/97vc2kZlIkurxXf0OIEnqLYtfkipj8UtSZSx+SaqMxS9JlVnW7wAnc+GFF+bIyEjb1/v2t7/Nueeee+oDnUaDlnnQ8oKZe2XQMg9aXlg884EDB76emS9bcEJmLtmf9evXZyfuu+++jq7XT4OWedDyZpq5VwYt86DlzVw8M/BAnqRbPdQjSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVWdJf2dArI9vvaWne4R1XneYkknT6uccvSZVZtPgj4vaIeDoiDjWN/UFEfDEiHo6IT0TEiqZ174iIqYh4PCJe2zS+sYxNRcT2U/9QJEmtaGWP/0PAxjlje4FLMvNHgC8B7wCIiIuBa4AfLtf504g4KyLOAj4AXAlcDFxb5kqSemzR4s/MzwLPzBn7dGaeKBf3A6vK8mZgIjP/IzO/DEwBl5Wfqcx8KjO/A0yUuZKkHovGN3guMiliBLg7My+ZZ93fAXdm5l9FxJ8A+zPzr8q6XcAny9SNmflLZfxNwOWZeeM8t7cN2AYwPDy8fmJiou0HNTMzw9DQUMvzD04fb2neupXL287SqnYz99ug5QUz98qgZR60vLB45vHx8QOZObrQ+q7O6omIdwIngDu6uZ1mmbkT2AkwOjqaY2Njbd/G5OQk7Vxva6tn9VzXfpZWtZu53wYtL5i5VwYt86Dlhe4zd1z8EbEVeB2wIf/vbcM0sLpp2qoyxknGJUk91NHpnBGxEXg78PrMfK5p1R7gmog4JyLWAGuBfwI+D6yNiDURcTaND4D3dBddktSJRff4I+IjwBhwYUQcAW6jcRbPOcDeiIDGcf1fzsxHIuKjwKM0DgHdkJn/VW7nRuBTwFnA7Zn5yGl4PJKkRSxa/Jl57TzDu04y/13Au+YZvxe4t610kqRTzr/claTKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVsfglqTIWvyRVxuKXpMpY/JJUGYtfkipj8UtSZSx+SaqMxS9JlbH4JakyFr8kVcbil6TKLFr8EXF7RDwdEYeaxs6PiL0R8UT5fV4Zj4h4f0RMRcTDEXFp03W2lPlPRMSW0/NwJEmLaWWP/0PAxjlj24F9mbkW2FcuA1wJrC0/24APQuOFArgNuBy4DLht9sVCktRbixZ/Zn4WeGbO8GZgd1neDVzdNP7hbNgPrIiIi4DXAnsz85nMfBbYy/9/MZEk9UBk5uKTIkaAuzPzknL5m5m5oiwH8GxmroiIu4Edmfm5sm4fcCswBrwoM3+vjP8m8Hxm/uE897WNxrsFhoeH109MTLT9oGZmZhgaGmp5/sHp4y3NW7dyedtZWtVu5n4btLxg5l4ZtMyDlhcWzzw+Pn4gM0cXWr+s2wCZmRGx+KtH67e3E9gJMDo6mmNjY23fxuTkJO1cb+v2e1qad/i69rO0qt3M/TZoecHMvTJomQctL3SfudOzeo6VQziU30+X8WlgddO8VWVsoXFJUo91Wvx7gNkzc7YAdzWNv7mc3XMFcDwzjwKfAl4TEeeVD3VfU8YkST226KGeiPgIjWP0F0bEERpn5+wAPhoR1wNfAd5Ypt8LbAKmgOeAtwBk5jMR8bvA58u838nMuR8YS5J6YNHiz8xrF1i1YZ65CdywwO3cDtzeVjpJ0innX+5KUmUsfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVsfglqTIWvyRVxuKXpMpY/JJUGYtfkipj8UtSZSx+SaqMxS9JlbH4JakyFr8kVcbil6TKWPySVBmLX5Iq01XxR8SvRcQjEXEoIj4SES+KiDURcX9ETEXEnRFxdpl7Trk8VdaPnIoHIElqT8fFHxErgV8FRjPzEuAs4Brg3cB7M/MHgGeB68tVrgeeLePvLfMkST3W7aGeZcCLI2IZ8BLgKPBTwMfK+t3A1WV5c7lMWb8hIqLL+5cktSkys/MrR9wEvAt4Hvg0cBOwv+zVExGrgU9m5iURcQjYmJlHyrongcsz8+tzbnMbsA1geHh4/cTERNu5ZmZmGBoaann+wenjLc1bt3J521la1W7mfhu0vGDmXhm0zIOWFxbPPD4+fiAzRxdav6zTO46I82jsxa8Bvgn8DbCx09ublZk7gZ0Ao6OjOTY21vZtTE5O0s71tm6/p6V5h69rP0ur2s3cb4OWF8zcK4OWedDyQveZuznU89PAlzPzXzPzP4GPA68CVpRDPwCrgOmyPA2sBijrlwPf6OL+JUkd6Kb4/wW4IiJeUo7VbwAeBe4D3lDmbAHuKst7ymXK+s9kN8eZJEkd6bj4M/N+Gh/SPggcLLe1E7gVuDkipoALgF3lKruAC8r4zcD2LnJLkjrU8TF+gMy8DbhtzvBTwGXzzP134Be6ub92jbR47F6SauJf7kpSZSx+SaqMxS9JlbH4JakyFr8kVcbil6TKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVsfglqTIWvyRVxuKXpMpY/JJUGYtfkipj8UtSZboq/ohYEREfi4gvRsRjEfHjEXF+ROyNiCfK7/PK3IiI90fEVEQ8HBGXnpqHIElqR7d7/O8D/j4zfwj4UeAxYDuwLzPXAvvKZYArgbXlZxvwwS7vW5LUgY6LPyKWA68GdgFk5ncy85vAZmB3mbYbuLosbwY+nA37gRURcVHHySVJHYnM7OyKET8G7AQepbG3fwC4CZjOzBVlTgDPZuaKiLgb2JGZnyvr9gG3ZuYDc253G413BAwPD6+fmJhoO9vMzAxDQ0McnD7e0WNbyLqVy0/p7TWbzTwoBi0vmLlXBi3zoOWFxTOPj48fyMzRhdYv6+K+lwGXAm/NzPsj4n3832EdADIzI6KtV5bM3EnjBYXR0dEcGxtrO9jk5CRjY2Ns3X5P29c9mcPXtZ+lVbOZB8Wg5QUz98qgZR60vNB95m6O8R8BjmTm/eXyx2i8EBybPYRTfj9d1k8Dq5uuv6qMSZJ6qOPiz8yvAV+NiFeUoQ00DvvsAbaUsS3AXWV5D/DmcnbPFcDxzDza6f1LkjrTzaEegLcCd0TE2cBTwFtovJh8NCKuB74CvLHMvRfYBEwBz5W5kqQe66r4M/MhYL4PEDbMMzeBG7q5P0lS9/zLXUmqjMUvSZWx+CWpMt1+uKsujDT9ncEt604s+HcHh3dc1atIkirgHr8kVcbil6TKWPySVBmLX5Iq44e7bRhp8Uvf/DBW0lLmHr8kVcbil6TKWPySVBmLX5IqY/FLUmU8q+c0aPXsH0nqB/f4JakyFr8kVcbil6TKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZWx+CWpMl0Xf0ScFRH/HBF3l8trIuL+iJiKiDsj4uwyfk65PFXWj3R735Kk9p2KPf6bgMeaLr8beG9m/gDwLHB9Gb8eeLaMv7fMkyT1WFfFHxGrgKuAvyiXA/gp4GNlym7g6rK8uVymrN9Q5kuSeqjbPf4/Bt4O/He5fAHwzcw8US4fAVaW5ZXAVwHK+uNlviSphyIzO7tixOuATZn5KxExBvw6sBXYXw7nEBGrgU9m5iURcQjYmJlHyrongcsz8+tzbncbsA1geHh4/cTERNvZZmZmGBoa4uD08Y4eWz8MvxiOPT//unUrl/c2TAtmt/EgMXNvDFrmQcsLi2ceHx8/kJmjC63v5muZXwW8PiI2AS8Cvgd4H7AiIpaVvfpVwHSZPw2sBo5ExDJgOfCNuTeamTuBnQCjo6M5NjbWdrDJyUnGxsbYOkBfj3zLuhO85+D8/zkOXzfW2zAtmN3Gg8TMvTFomQctL3SfueNDPZn5jsxclZkjwDXAZzLzOuA+4A1l2hbgrrK8p1ymrP9Mdvp2Q5LUsdNxHv+twM0RMUXjGP6uMr4LuKCM3wxsPw33LUlaxCn5F7gycxKYLMtPAZfNM+ffgV84FfcnSeqcf7krSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVsfglqTIWvyRVxuKXpMpY/JJUGYtfkipj8UtSZSx+SaqMxS9JlbH4JakyFr8kVcbil6TKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZXpuPgjYnVE3BcRj0bEIxFxUxk/PyL2RsQT5fd5ZTwi4v0RMRURD0fEpafqQUiSWresi+ueAG7JzAcj4qXAgYjYC2wF9mXmjojYDmwHbgWuBNaWn8uBD5bfWsTI9ntannt4x1WnMYmkM0HHe/yZeTQzHyzL/wY8BqwENgO7y7TdwNVleTPw4WzYD6yIiIs6Ti5J6khkZvc3EjECfBa4BPiXzFxRxgN4NjNXRMTdwI7M/FxZtw+4NTMfmHNb24BtAMPDw+snJibazjMzM8PQ0BAHp493/qB6bPjFcOz57m9n3crl3d9IC2a38SAxc28MWuZBywuLZx4fHz+QmaMLre/mUA8AETEE/C3wtsz8VqPrGzIzI6KtV5bM3AnsBBgdHc2xsbG2M01OTjI2NsbWNg6R9Nst607wnoNd/+fg8HVj3Ydpwew2HiRm7o1ByzxoeaH7zF2d1RMR302j9O/IzI+X4WOzh3DK76fL+DSwuunqq8qYJKmHujmrJ4BdwGOZ+UdNq/YAW8ryFuCupvE3l7N7rgCOZ+bRTu9fktSZbo4tvAp4E3AwIh4qY78B7AA+GhHXA18B3ljW3QtsAqaA54C3dHHfkqQOdVz85UPaWGD1hnnmJ3BDp/cnSTo1/MtdSaqMxS9JlbH4JakyFr8kVcbil6TKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5Jqkw3//SilqCR7fe0NO/wjqtOcxJJS5V7/JJUGYtfkipj8UtSZSx+SaqMxS9JlbH4Jakyns5ZKU/7lOrV8+KPiI3A+4CzgL/IzB29zqD+8QVH6r+eHuqJiLOADwBXAhcD10bExb3MIEm16/Ue/2XAVGY+BRARE8Bm4NEe51CLFtpDv2XdCba2uPd+OrX6DgJaz9zqu4127rsVvstRr0Rm9u7OIt4AbMzMXyqX3wRcnpk3Ns3ZBmwrF18BPN7BXV0IfL3LuL02aJkHLS+YuVcGLfOg5YXFM39fZr5soZVL7sPdzNwJ7OzmNiLigcwcPUWRemLQMg9aXjBzrwxa5kHLC91n7vXpnNPA6qbLq8qYJKlHel38nwfWRsSaiDgbuAbY0+MMklS1nh7qycwTEXEj8Ckap3PenpmPnIa76upQUZ8MWuZBywtm7pVByzxoeaHbw+G9/HBXktR/fmWDJFXG4pekypxRxR8RGyPi8YiYiojt/c4zn4hYHRH3RcSjEfFIRNxUxn87IqYj4qHys6nfWZtFxOGIOFiyPVDGzo+IvRHxRPl9Xr9zzoqIVzRty4ci4lsR8baltp0j4vaIeDoiDjWNzbtdo+H95fn9cERcukTy/kFEfLFk+kRErCjjIxHxfNO2/rNe5z1J5gWfBxHxjrKNH4+I1y6hzHc25T0cEQ+V8fa3c2aeET80Pix+Eng5cDbwBeDifueaJ+dFwKVl+aXAl2h8fcVvA7/e73wnyX0YuHDO2O8D28vyduDd/c55kufG14DvW2rbGXg1cClwaLHtCmwCPgkEcAVw/xLJ+xpgWVl+d1PekeZ5S2wbz/s8KP8vfgE4B1hTOuWspZB5zvr3AL/V6XY+k/b4//frIDLzO8Ds10EsKZl5NDMfLMv/BjwGrOxvqo5tBnaX5d3A1X3McjIbgCcz8yv9DjJXZn4WeGbO8ELbdTPw4WzYD6yIiIt6k7RhvryZ+enMPFEu7qfx9zlLxgLbeCGbgYnM/I/M/DIwRaNbeupkmSMigDcCH+n09s+k4l8JfLXp8hGWeKFGxAjwSuD+MnRjebt8+1I6bFIk8OmIOFC+VgNgODOPluWvAcP9ibaoa3jh/yRLeTvDwtt1EJ7jv0jjXcmsNRHxzxHxjxHxk/0KtYD5ngeDsI1/EjiWmU80jbW1nc+k4h8oETEE/C3wtsz8FvBB4PuBHwOO0ngrt5T8RGZeSuObVW+IiFc3r8zGe84ld25w+UPB1wN/U4aW+nZ+gaW6XecTEe8ETgB3lKGjwPdm5iuBm4G/jojv6Ve+OQbqeTDHtbxwR6bt7XwmFf/AfB1ERHw3jdK/IzM/DpCZxzLzvzLzv4E/pw9vL08mM6fL76eBT9DId2z2UEP5/XT/Ei7oSuDBzDwGS387Fwtt1yX7HI+IrcDrgOvKixXlcMk3yvIBGsfLf7BvIZuc5HmwZLcxQEQsA34OuHN2rJPtfCYV/0B8HUQ5PrcLeCwz/6hpvPlY7c8Ch+Zet18i4tyIeOnsMo0P8w7R2L5byrQtwF39SXhSL9g7WsrbuclC23UP8OZyds8VwPGmQ0J9E41/XOntwOsz87mm8ZdF49/gICJeDqwFnupPyhc6yfNgD3BNRJwTEWtoZP6nXuc7iZ8GvpiZR2YHOtrOvf60+jR/Er6JxlkyTwLv7HeeBTL+BI237g8DD5WfTcBfAgfL+B7gon5nbcr8chpnOnwBeGR22wIXAPuAJ4B/AM7vd9Y5uc8FvgEsbxpbUtuZxovSUeA/aRxPvn6h7UrjbJ4PlOf3QWB0ieSdonFcfPb5/Gdl7s+X58tDwIPAzyyhbbzg8wB4Z9nGjwNXLpXMZfxDwC/Pmdv2dvYrGySpMmfSoR5JUgssfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klSZ/wEmDJk6BAzVDQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3cIhTGKT_DoJ",
        "outputId": "b5f17d16-7526-47cf-ddb4-6a1651d98669"
      },
      "source": [
        "# tokenize and encode sequences in the training set\n",
        "tokens_train = tokenizer.batch_encode_plus(\n",
        "    train_text.tolist(),\n",
        "    max_length = 25,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True\n",
        ")\n",
        "\n",
        "# tokenize and encode sequences in the validation set\n",
        "tokens_val = tokenizer.batch_encode_plus(\n",
        "    val_text.tolist(),\n",
        "    max_length = 25,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True\n",
        ")\n",
        "\n",
        "# tokenize and encode sequences in the test set\n",
        "tokens_test = tokenizer.batch_encode_plus(\n",
        "    test_text.tolist(),\n",
        "    max_length = 25,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True\n",
        ")"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2204: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QcEJ5vrV_Dqt"
      },
      "source": [
        "## convert lists to tensors\n",
        "\n",
        "train_seq = torch.tensor(tokens_train['input_ids'])\n",
        "train_mask = torch.tensor(tokens_train['attention_mask'])\n",
        "train_y = torch.tensor(train_labels.tolist())\n",
        "\n",
        "val_seq = torch.tensor(tokens_val['input_ids'])\n",
        "val_mask = torch.tensor(tokens_val['attention_mask'])\n",
        "val_y = torch.tensor(val_labels.tolist())\n",
        "\n",
        "test_seq = torch.tensor(tokens_test['input_ids'])\n",
        "test_mask = torch.tensor(tokens_test['attention_mask'])\n",
        "test_y = torch.tensor(test_labels.tolist())"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4FAZF1i4_DtE"
      },
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "#define a batch size\n",
        "batch_size = 32\n",
        "\n",
        "# wrap tensors\n",
        "train_data = TensorDataset(train_seq, train_mask, train_y)\n",
        "\n",
        "# sampler for sampling the data during training\n",
        "train_sampler = RandomSampler(train_data)\n",
        "\n",
        "# dataLoader for train set\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "# wrap tensors\n",
        "val_data = TensorDataset(val_seq, val_mask, val_y)\n",
        "\n",
        "# sampler for sampling the data during training\n",
        "val_sampler = SequentialSampler(val_data)\n",
        "\n",
        "# dataLoader for validation set\n",
        "val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size=batch_size)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lR7jgEyh_4v4"
      },
      "source": [
        "model = FastTransformer(\n",
        "    num_tokens = 30522,\n",
        "    dim = 768,\n",
        "    depth = 2,\n",
        "    max_seq_len = 512,\n",
        "    absolute_pos_emb = True   # default uses relative positional encoding, but if that isn't working, then turn on absolute positional embedding by setting this to True\n",
        ")\n",
        "x = torch.randint(0, 30522, (1, 512))\n",
        "mask = torch.ones(1, 512).bool()\n",
        "\n",
        "logits = model(x, mask = mask)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lPvVpaQv_Dvc"
      },
      "source": [
        "# optimizer from hugging face transformers\n",
        "from transformers import AdamW\n",
        "\n",
        "# define the optimizer\n",
        "optimizer = AdamW(model.parameters(),lr = 1e-5)          # learning rate"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HawNTbLx_DyA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b1ff1d4a-2d7e-4ff6-e619-0e4cc14e5d26"
      },
      "source": [
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")\n",
        "model = model.to(device)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "No GPU available, using the CPU instead.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FWeSyDnp_D0Y",
        "outputId": "ac3acb5e-a867-476f-9748-5c75cf570db7"
      },
      "source": [
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "\n",
        "#compute the class weights\n",
        "class_weights = compute_class_weight('balanced', np.unique(train_labels), train_labels)\n",
        "\n",
        "print(\"Class Weights:\",class_weights)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Class Weights: [0.57743559 3.72848948]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a2VyAmkb_D2x"
      },
      "source": [
        "# converting list of class weights to a tensor\n",
        "weights= torch.tensor(class_weights,dtype=torch.float)\n",
        "\n",
        "# push to GPU\n",
        "weights = weights.to(device)\n",
        "\n",
        "# define the loss function\n",
        "cross_entropy  = nn.NLLLoss(weight=weights) \n",
        "\n",
        "# number of training epochs\n",
        "epochs = 1"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FvNXQnDQALwM"
      },
      "source": [
        "# function to train the model\n",
        "def train():\n",
        "  \n",
        "  model.train()\n",
        "\n",
        "  total_loss, total_accuracy = 0, 0\n",
        "  \n",
        "  # empty list to save model predictions\n",
        "  total_preds=[]\n",
        "  \n",
        "  # iterate over batches\n",
        "  for step,batch in enumerate(train_dataloader):\n",
        "    \n",
        "    # progress update after every 50 batches.\n",
        "    if step % 50 == 0 and not step == 0:\n",
        "      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))\n",
        "\n",
        "    # push the batch to gpu\n",
        "    batch = [r.to(device) for r in batch]\n",
        " \n",
        "    sent_id, mask, labels = batch\n",
        "\n",
        "    # clear previously calculated gradients \n",
        "    model.zero_grad()        \n",
        "\n",
        "    # get model predictions for the current batch\n",
        "    preds = model(sent_id, mask)\n",
        "\n",
        "    # compute the loss between actual and predicted values\n",
        "    loss = cross_entropy(preds, labels)\n",
        "\n",
        "    # add on to the total loss\n",
        "    total_loss = total_loss + loss.item()\n",
        "\n",
        "    # backward pass to calculate the gradients\n",
        "    loss.backward()\n",
        "\n",
        "    # clip the the gradients to 1.0. It helps in preventing the exploding gradient problem\n",
        "    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "    # update parameters\n",
        "    optimizer.step()\n",
        "\n",
        "    # model predictions are stored on GPU. So, push it to CPU\n",
        "    preds=preds.detach().cpu().numpy()\n",
        "\n",
        "    # append the model predictions\n",
        "    total_preds.append(preds)\n",
        "\n",
        "  # compute the training loss of the epoch\n",
        "  avg_loss = total_loss / len(train_dataloader)\n",
        "  \n",
        "  # predictions are in the form of (no. of batches, size of batch, no. of classes).\n",
        "  # reshape the predictions in form of (number of samples, no. of classes)\n",
        "  total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "  #returns the loss and predictions\n",
        "  return avg_loss, total_preds"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YC2r8TYmALzG"
      },
      "source": [
        "# function for evaluating the model\n",
        "def evaluate():\n",
        "  \n",
        "  print(\"\\nEvaluating...\")\n",
        "  \n",
        "  # deactivate dropout layers\n",
        "  model.eval()\n",
        "\n",
        "  total_loss, total_accuracy = 0, 0\n",
        "  \n",
        "  # empty list to save the model predictions\n",
        "  total_preds = []\n",
        "\n",
        "  # iterate over batches\n",
        "  for step,batch in enumerate(val_dataloader):\n",
        "    \n",
        "    # Progress update every 50 batches.\n",
        "    if step % 50 == 0 and not step == 0:\n",
        "      \n",
        "      # Calculate elapsed time in minutes.\n",
        "      elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "      # Report progress.\n",
        "      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(val_dataloader)))\n",
        "\n",
        "    # push the batch to gpu\n",
        "    batch = [t.to(device) for t in batch]\n",
        "\n",
        "    sent_id, mask, labels = batch\n",
        "\n",
        "    # deactivate autograd\n",
        "    with torch.no_grad():\n",
        "      \n",
        "      # model predictions\n",
        "      preds = model(sent_id, mask)\n",
        "\n",
        "      # compute the validation loss between actual and predicted values\n",
        "      loss = cross_entropy(preds,labels)\n",
        "\n",
        "      total_loss = total_loss + loss.item()\n",
        "\n",
        "      preds = preds.detach().cpu().numpy()\n",
        "\n",
        "      total_preds.append(preds)\n",
        "\n",
        "  # compute the validation loss of the epoch\n",
        "  avg_loss = total_loss / len(val_dataloader) \n",
        "\n",
        "  # reshape the predictions in form of (number of samples, no. of classes)\n",
        "  total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "  return avg_loss, total_preds"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "id": "9vi3ogyAAL1g",
        "outputId": "514f8f73-1452-4622-d0a1-6436da39e558"
      },
      "source": [
        "\n",
        "#device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "# set initial loss to infinite\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "# empty lists to store training and validation loss of each epoch\n",
        "train_losses=[]\n",
        "valid_losses=[]\n",
        "\n",
        "#for each epoch\n",
        "for epoch in range(epochs):\n",
        "     \n",
        "    print('\\n Epoch {:} / {:}'.format(epoch + 1, epochs))\n",
        "    \n",
        "    #train model\n",
        "    train_loss, _ = train()\n",
        "    \n",
        "    #evaluate model\n",
        "    valid_loss, _ = evaluate()\n",
        "    \n",
        "    #save the best model\n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'saved_weights.pt')\n",
        "    \n",
        "    # append training and validation loss\n",
        "    train_losses.append(train_loss)\n",
        "    valid_losses.append(valid_loss)\n",
        "    \n",
        "    \n",
        "    print(f'\\nTraining Loss: {train_loss:.3f}')\n",
        "    print(f'Validation Loss: {valid_loss:.3f}')"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Epoch 1 / 1\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-27-d26b9ee79f1a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0;31m#train model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0mtrain_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0;31m#evaluate model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-a8875e82e2a3>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0;31m# get model predictions for the current batch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m     \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0;31m# compute the loss between actual and predicted values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/fast_transformer_pytorch/fast_transformer_pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, mask)\u001b[0m\n\u001b[1;32m    210\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrearrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpos_emb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'n d -> () n d'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 212\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mattn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mff\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    213\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mattn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    214\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mff\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/fast_transformer_pytorch/fast_transformer_pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;31m# blocks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/fast_transformer_pytorch/fast_transformer_pytorch.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, mask)\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;31m# calculate query attention logits\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m         \u001b[0mq_attn_logits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrearrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_q_attn_logits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'b h n () -> b h n'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscale\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m         \u001b[0mq_attn_logits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mq_attn_logits\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmasked_fill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m~\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask_value\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m         \u001b[0mq_attn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mq_attn_logits\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: Mask tensor can take 0 and 1 values only"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q5iLbkEnBhcm"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9QrANmMFECkM",
        "outputId": "83e9191a-2d91-4f1c-fa2b-63a5c94f4c9e"
      },
      "source": [
        "model.parameters()\n",
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The Fast transformer  model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The Fast transformer  model has 34 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "token_emb.weight                                        (30522, 768)\n",
            "abs_pos_emb.weight                                        (512, 768)\n",
            "layers.0.0.norm.weight                                        (768,)\n",
            "layers.0.0.norm.bias                                          (768,)\n",
            "layers.0.0.fn.to_qkv.weight                              (1536, 768)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "layers.0.0.fn.to_q_attn_logits.weight                        (1, 64)\n",
            "layers.0.0.fn.to_k_attn_logits.weight                        (1, 64)\n",
            "layers.0.0.fn.to_r.weight                                   (64, 64)\n",
            "layers.0.0.fn.to_r.bias                                        (64,)\n",
            "layers.0.0.fn.to_out.weight                               (768, 512)\n",
            "layers.0.0.fn.to_out.bias                                     (768,)\n",
            "layers.0.1.norm.weight                                        (768,)\n",
            "layers.0.1.norm.bias                                          (768,)\n",
            "layers.0.1.fn.0.weight                                   (3072, 768)\n",
            "layers.0.1.fn.0.bias                                         (3072,)\n",
            "layers.0.1.fn.2.weight                                   (768, 3072)\n",
            "layers.0.1.fn.2.bias                                          (768,)\n",
            "layers.1.0.norm.weight                                        (768,)\n",
            "layers.1.0.norm.bias                                          (768,)\n",
            "layers.1.0.fn.to_qkv.weight                              (1536, 768)\n",
            "layers.1.0.fn.to_r.weight                                   (64, 64)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "to_logits.0.weight                                            (768,)\n",
            "to_logits.0.bias                                              (768,)\n",
            "to_logits.1.weight                                      (30522, 768)\n",
            "to_logits.1.bias                                            (30522,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_tbwCXXVMYpd"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}