jax.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/benjaminbeilharz/annotated-transformer/blob/main/jax.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MXfXODBNAVs1",
        "outputId": "5f388c3f-0a8d-4360-adb1-364fd860e4f5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: jax in /usr/local/lib/python3.7/dist-packages (0.3.4)\n",
            "Requirement already satisfied: optax in /usr/local/lib/python3.7/dist-packages (0.1.1)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (2.0.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from jax) (3.10.0.2)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.7/dist-packages (from jax) (1.0.0)\n",
            "Requirement already satisfied: scipy>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from jax) (1.4.1)\n",
            "Requirement already satisfied: opt-einsum in /usr/local/lib/python3.7/dist-packages (from jax) (3.3.0)\n",
            "Requirement already satisfied: numpy>=1.19 in /usr/local/lib/python3.7/dist-packages (from jax) (1.21.5)\n",
            "Requirement already satisfied: chex>=0.0.4 in /usr/local/lib/python3.7/dist-packages (from optax) (0.1.1)\n",
            "Requirement already satisfied: jaxlib>=0.1.37 in /usr/local/lib/python3.7/dist-packages (from optax) (0.3.2+cuda11.cudnn805)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py->jax) (1.15.0)\n",
            "Requirement already satisfied: dm-tree>=0.1.5 in /usr/local/lib/python3.7/dist-packages (from chex>=0.0.4->optax) (0.1.6)\n",
            "Requirement already satisfied: toolz>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from chex>=0.0.4->optax) (0.11.2)\n",
            "Requirement already satisfied: flatbuffers<3.0,>=1.12 in /usr/local/lib/python3.7/dist-packages (from jaxlib>=0.1.37->optax) (2.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2022.2.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.11.3)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.18.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.63.0)\n",
            "Requirement already satisfied: pyarrow>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.4.0)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.13)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.7)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.25.11)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.7.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.4.0)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.7.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n"
          ]
        }
      ],
      "source": [
        "!pip install --upgrade jax optax datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BEQoFjLZwucm"
      },
      "outputs": [],
      "source": [
        "from typing import *\n",
        "from tqdm import tqdm\n",
        "\n",
        "# jax\n",
        "import jax\n",
        "import jax.numpy as jnp\n",
        "import jax.random as random\n",
        "\n",
        "# gradient processing + optimizers for the jax ecosystem\n",
        "import optax as O\n",
        "\n",
        "# data\n",
        "from datasets import load_dataset\n",
        "\n",
        "# enable colab tpu\n",
        "import jax.tools.colab_tpu\n",
        "jax.tools.colab_tpu.setup_tpu()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# An Introduction to JAX\n",
        "\n",
        "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAPoAAACRCAYAAADq36WkAAAMbmlDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkJDQAghICb0jUgNICaEFkF4EGyEJJJQYE4KKHV1UcO0iihVdFVFsKyAWELuyKPa+WFBR1kVdbKi8CQnouq9873zf3PvnzJn/lDuTew8Amh+4EkkeqgVAvrhAmhAezBiTls4gPQUooAEKcAJ2XJ5MwoqLiwZQBu9/l3c3AKK4X3VWcP1z/r+KDl8g4wGAjIM4ky/j5UPcDAC+nieRFgBAVOgtpxRIFHgOxLpSGCDEqxQ4W4l3KnCmEh8dsElKYEN8GQA1KpcrzQZA4x7UMwp52ZBH4zPErmK+SAyAphPEATwhlw+xInan/PxJClwBsR20l0AM4wHMzO84s//GnznEz+VmD2FlXgOiFiKSSfK40/7P0vxvyc+TD/qwgYMqlEYkKPKHNbyVOylKgakQd4szY2IVtYb4g4ivrDsAKEUoj0hW2qPGPBkb1g/oQ+zK54ZEQWwMcZg4LyZapc/MEoVxIIa7BZ0qKuAkQWwA8UKBLDRRZbNZOilB5Quty5KyWSr9Oa50wK/C1wN5bjJLxf9GKOCo+DGNImFSKsQUiK0KRSkxEGtA7CLLTYxS2YwqErJjBm2k8gRF/FYQJwjE4cFKfqwwSxqWoLIvzZcN5ottFoo4MSp8oECYFKGsD3aKxx2IH+aCXRaIWcmDPALZmOjBXPiCkFBl7thzgTg5UcXzQVIQnKBci1MkeXEqe9xCkBeu0FtA7CErTFStxVMK4OZU8uNZkoK4JGWceFEONzJOGQ++DEQDNggBDCCHIxNMAjlA1NZd3w1/KWfCABdIQTYQAGeVZnBF6sCMGF4TQRH4AyIBkA2tCx6YFYBCqP8ypFVenUHWwGzhwIpc8BTifBAF8uBv+cAq8ZC3FPAEakT/8M6FgwfjzYNDMf/v9YPabxoW1ESrNPJBjwzNQUtiKDGEGEEMI9rjRngA7odHw2sQHG44E/cZzOObPeEpoZ3wiHCd0EG4PVFULP0hytGgA/KHqWqR+X0tcBvI6YkH4/6QHTLj+rgRcMY9oB8WHgg9e0ItWxW3oiqMH7j/lsF3T0NlR3Ylo+Rh5CCy3Y8rNRw0PIdYFLX+vj7KWDOH6s0emvnRP/u76vPhPepHS2whdhA7i53AzmNHsXrAwJqwBqwVO6bAQ7vrycDuGvSWMBBPLuQR/cMfV+VTUUmZa41rl+tn5VyBYGqB4uCxJ0mmSUXZwgIGC74dBAyOmOfixHBzdXMDQPGuUf59vY0feIcg+q3fdPN+B8C/qb+//8g3XWQTAPu94fE//E1nxwRAWx2Ac4d5cmmhUocrLgT4L6EJT5ohMAWWwA7m4wa8gB8IAqEgEsSCJJAGJsAqC+E+l4IpYAaYC0pAGVgGVoN1YBPYCnaCPeAAqAdHwQlwBlwEl8F1cBfunk7wEvSAd6APQRASQkPoiCFihlgjjogbwkQCkFAkGklA0pAMJBsRI3JkBjIPKUNWIOuQLUg1sh85jJxAziPtyG3kIdKFvEE+oRhKRXVRE9QGHYEyURYahSah49FsdDJahM5Hl6AVaBW6G61DT6AX0etoB/oS7cUApo7pY+aYM8bE2Fgslo5lYVJsFlaKlWNVWC3WCJ/zVawD68Y+4kScjjNwZ7iDI/BknIdPxmfhi/F1+E68Dj+FX8Uf4j34VwKNYExwJPgSOIQxhGzCFEIJoZywnXCIcBqepU7COyKRqE+0JXrDs5hGzCFOJy4mbiDuJTYT24mPib0kEsmQ5EjyJ8WSuKQCUglpLWk3qYl0hdRJ+qCmrmam5qYWppauJlYrVitX26V2XO2K2jO1PrIW2ZrsS44l88nTyEvJ28iN5EvkTnIfRZtiS/GnJFFyKHMpFZRaymnKPcpbdXV1C3Uf9Xh1kfoc9Qr1fern1B+qf6TqUB2obOo4qpy6hLqD2ky9TX1Lo9FsaEG0dFoBbQmtmnaS9oD2QYOu4aLB0eBrzNao1KjTuKLxSpOsaa3J0pygWaRZrnlQ85JmtxZZy0aLrcXVmqVVqXVY66ZWrzZde6R2rHa+9mLtXdrntZ/rkHRsdEJ1+DrzdbbqnNR5TMfolnQ2nUefR99GP03v1CXq2upydHN0y3T36Lbp9ujp6HnopehN1avUO6bXoY/p2+hz9PP0l+of0L+h/2mYyTDWMMGwRcNqh10Z9t5guEGQgcCg1GCvwXWDT4YMw1DDXMPlhvWG941wIwejeKMpRhuNTht1D9cd7jecN7x0+IHhd4xRYwfjBOPpxluNW417TUxNwk0kJmtNTpp0m+qbBpnmmK4yPW7aZUY3CzATma0yazJ7wdBjsBh5jArGKUaPubF5hLncfIt5m3mfha1FskWxxV6L+5YUS6ZlluUqyxbLHiszq9FWM6xqrO5Yk62Z1kLrNdZnrd/b2Nqk2iywqbd5bmtgy7Etsq2xvWdHswu0m2xXZXfNnmjPtM+132B/2QF18HQQOlQ6XHJEHb0cRY4bHNudCE4+TmKnKqebzlRnlnOhc43zQxd9l2iXYpd6l1cjrEakj1g+4uyIr66ernmu21zvjtQZGTmyeGTjyDduDm48t0q3a+409zD32e4N7q89HD0EHhs9bnnSPUd7LvBs8fzi5e0l9ar16vK28s7wXu99k6nLjGMuZp7zIfgE+8z2Oerz0dfLt8D3gO+ffs5+uX67/J6Psh0lGLVt1GN/C3+u/xb/jgBGQEbA5oCOQPNAbmBV4KMgyyB+0PagZyx7Vg5rN+tVsGuwNPhQ8Hu2L3smuzkECwkPKQ1pC9UJTQ5dF/ogzCIsO6wmrCfcM3x6eHMEISIqYnnETY4Jh8ep5vREekfOjDwVRY1KjFoX9SjaIVoa3TgaHR05euXoezHWMeKY+lgQy4ldGXs/zjZuctyReGJ8XHxl/NOEkQkzEs4m0hMnJu5KfJcUnLQ06W6yXbI8uSVFM2VcSnXK+9SQ1BWpHWNGjJk55mKaUZoorSGdlJ6Svj29d2zo2NVjO8d5jisZd2O87fip489PMJqQN+HYRM2J3IkHMwgZqRm7Mj5zY7lV3N5MTub6zB4em7eG95IfxF/F7xL4C1YInmX5Z63Iep7tn70yu0sYKCwXdovYonWi1zkROZty3ufG5u7I7c9Lzdubr5afkX9YrCPOFZ+aZDpp6qR2iaOkRNIx2Xfy6sk90ijpdhkiGy9rKNCFH/Wtcjv5T/KHhQGFlYUfpqRMOThVe6p4aus0h2mLpj0rCiv6ZTo+nTe9ZYb5jLkzHs5kzdwyC5mVOatltuXs+bM754TP2TmXMjd37m/FrsUriv+alzqvcb7J/DnzH/8U/lNNiUaJtOTmAr8FmxbiC0UL2xa5L1q76Gspv/RCmWtZednnxbzFF34e+XPFz/1Lspa0LfVaunEZcZl42Y3lgct3rtBeUbTi8crRK+tWMVaVrvpr9cTV58s9yjetoayRr+moiK5oWGu1dtnaz+uE665XBlfuXW+8ftH69xv4G65sDNpYu8lkU9mmT5tFm29tCd9SV2VTVb6VuLVw69NtKdvO/sL8pXq70fay7V92iHd07EzYearau7p6l/GupTVojbyma/e43Zf3hOxpqHWu3bJXf2/ZPrBPvu/F/oz9Nw5EHWg5yDxY+6v1r+sP0Q+V1iF10+p66oX1HQ1pDe2HIw+3NPo1HjricmTHUfOjlcf0ji09Tjk+/3h/U1FTb7OkuftE9onHLRNb7p4cc/LaqfhTbaejTp87E3bm5FnW2aZz/ueOnvc9f/gC80L9Ra+Lda2erYd+8/ztUJtXW90l70sNl30uN7aPaj9+JfDKiashV89c41y7eD3mevuN5Bu3bo672XGLf+v57bzbr+8U3um7O+ce4V7pfa375Q+MH1T9bv/73g6vjmMPQx62Pkp8dPcx7/HLJ7InnzvnP6U9LX9m9qz6udvzo11hXZdfjH3R+VLysq+75A/tP9a/snv1659Bf7b2jOnpfC193f9m8VvDtzv+8virpTeu98G7/Hd970s/GH7Y+ZH58eyn1E/P+qZ8Jn2u+GL/pfFr1Nd7/fn9/RKulDvwKYDBgWZlAfBmBwC0NADosG+jjFX2ggOCKPvXAQT+E1b2iwPiBUAt/H6P74ZfNzcB2LcNtl+QXxP2qnE0AJJ8AOruPjRUIstyd1NyUWGfQnjQ3/8W9myklQB8Wdbf31fV3/9lKwwW9o7NYmUPqhAi7Bk2h37JzM8E/0aU/el3Of54B4oIPMCP938Bk5OQ0jwxNzAAAAA4ZVhJZk1NACoAAAAIAAGHaQAEAAAAAQAAABoAAAAAAAKgAgAEAAAAAQAAAPqgAwAEAAAAAQAAAJEAAAAAYF6JVQAAQABJREFUeAHsfQdgXNWV9nnTR6NRG416dZXlLhcMBtsCEgghsMACKUvaJoTdJGwghA6WqTYhdYGEze7mJ0tYAgGSQCAYG8tgGxtbsmyr9zIaldGoa/rM+7/zRmNkoTIjzUvI/+uCPDOv3HLuPfece6pAC+X/CwiIoqgSBMHX2Nr6/dY+w5NOtzgkCKJKtsEHiESVSEuNXjL06EnEf7IVATX7Bb8Y64/zZzveXpSZfznaEwQSZGk0BMv6jqYv6izG39KYYpiUolK2IWKiyE+CP8+poARfcX5y/jH0QYn59IcLU/kmOtweLDwnOwSwKBSM5KJYkfnWUd1DT58wkUb0xBNQIdqFMUujIOoYU9D1q8ZosyWWVO8bSIwlEoD8chQRw1D6iOzndVNf98hlrR0dXxKyhd+eOHFCvXHjRm802wQsAUrBd0J8PUZ7LHZP0ttmcjgDcUoFBi1DwfZBwhBRoMgb6BjtU4iJ3ifQzHb0ISJoLiC6DJPzSavyZawV7lNVQ+IjJ3tTY/Tk8sTrRLV/Ar0LofyES8GX8N7EaxPHNvGd0He+H/ALVJDqpcviPKT+MJa8JpAjfmC6iiZWGul3JSodVJAIfG4ZaQ40fbVXufHdJY8CIV8BMrjGETN6LZcSUI98CW2r70ltS8kacYoeMgKWgbCJa/gjBMxETJIyQ0l9uVbh8MWnfKv/fem2jn7rF7OTMl6IZCNbQPTwwf53+SQWusTiDQ/XXLD3VOJX32t2i5kxotoL3JNjQCpwmb1ugb5W6KDUBgN5nWCgDSA+QH45ighKrkwQqDexi9p+1Qu0D3jtBx25TYva7kN7D5SVlfEajwpVf+mll5RCseCrEsuWaN6NvWusHJuawa8WfDKwRgwshUjCiIK8RWNUd6KJVKQS2p7pppgVSt7IXo1kI5OH3+BOLpS/OQTGqZlEaqrbjE8cajdSojrg9wfkQXIFcNnhUdDKTA9t8oPK1uqIGMkDMiE5IwKOCF4cERpONZN3X4C056tVLbss1HWi706XOLSEWXfAIarrXF+b8lhKk0kdUAS8AosC5CioVcSGqcgVqVPRTl13DpB2k1rpqHF4Bw+585rbW+/mZsc3sll7EFUAzNrawgN/bQgwm0mWrsYv19sztlb3uH0xahaRyVNY9OUA8n0mw0XxlbEUAC0FIsjUGKp1ARFyiKzKDrLePUCaFSoSRwLAPMFrf92haars2c2Nv/zyy/PuxIEDB1Q33HCDv3qw+tL4tpTrR+rEgBgDYaZMwGThpcKnoLHlg1S/r5UUODGI7gBp1qiULSWdZD3ed5dTHMznjYw5jdmAvMC6zwahv9P7JSUl4wK412MOlBke3ltPlKYPCD6ZqKsSqDQExNu62EmFgxrytWuIEsBMyMWyY1dReJXkLBig+r1ABHAQkDCS6CBSbVCqO5/r8ZuKjddZR62fzojN2Bs6wsxlOsc5IxwSiPRNCbt1ZzTkMAQCFAA3LUeRdkwAtMBHrQMtNPBTB+k2aLCJgYPRAudJ9Pa/5dK1FPQ+iua/aK42z7qRLVB0OSbqE1DnlVdeKe3yTW0r767qS8sZcni8QMZZd/65dh3LnjQxAbok0UvailgSwbILMiG5pDUbw9pe4afWoVYa+JGDVOtA8ZzoPaPBmEiaHLXY/qyNek4N7OEx4TzrZ4Sdy/hC7HG9tf5msyV9g6NH9Ikq8CtyUXPUq9AqaDCrl5p+ZyGNUk2iCxfRe9FNpC5Sqiz/1R2wlQ1/odfbu624pNh3QDww46azgOhzmflP+DvMygXPpuW51RbDHfsbvJSsCygnStmjOQQWwNndCro030m5nXryDipJkO2AgMWOzivVChrOsVHTqxZSC8A5XxARpHFh01EkC6rBD4a8Q8fd61qs7beMjzfija5ELFEwLA+JfzRqWxJ2+k6i/ViwDzJxRpIAbhTVr3JRY3MzuV7zk2I9MDwkTsTXgFMUNBkq0fqbfrKW9UkbWbFQPKPYfwHRo7niPyF1VZuDrFxlvQnqtBS9UvQzGsgy1yyKcnsVlG7y0lZNgBRnYohi5RPAhSTRgdVuamptJseL3iAieCYAnykfqLp2pVrZ9L1O6jndXwJqngCqDluCyARzV5YFOaPU1jX3mVvNGT6niAYFWWApUWwgtCKZqDfOSu3P2EiTh00MYzlHfAqUVqQqlP37h30DHzq3tNs7v4bRi6xumwCFc77OSO7PeXLhx98FBLCQJQu44f6qi/5yJuGfjra7xDS9qISkXZbC5mC9YNGvy3FSCtRpPi9EYUB4uSiepE4zEfUkWqn1yV7S5AMRnJMQYXykglpQeLANDR10pzYuabkfl+8oLS1lJA0LGoAlqya91eLJpbp98bc5zuBFg18l29hY7uBUkmfzCDWcaKHAKcgbigQSRyeNjzcyBzayQpWi5R4rGQqUj6Cvv0dfR/CJDz7kn1vk2ZnObWPh118RAqCwEgtX05mw5/22OKjTxADOz1ga0S8sgBsBy742001FEIyJ9fKq0yRq7lCQB+q0uhPN5D8O4VQiOjEN08rIoFmvVjU/boGU2n7biDhSWFxczFQ9PBYelkZctDXJe0yNiRpRGfBJlrXBy9H9l2eIbQ7yA2QJtFP3/YOkWTUFNZ/QKm9k7jG3d+CgO6OpuZU3MioNGvRMeCr4dQHRPwaSv98LQdZNELu7m75ebUs7v8nu9mlhg/2x7T1aQ0TFLpzFL0t3UxzUaX4wjnKq00RWp+WJZBHbqPteIMJqIMJkajdxbIw8XhF8NtRtf3IoWk92Suq2XcKuWUHCwi3hBsFfO1R7eUJb6jWOJtio6eSzZ5fUaThVjC0ZoIa9bZCaYi/imZuhp8zJSBvZo9jITtjvwEa2ohgGPVOp2xZY94kL4+/4+zjLBuOQN+P2nzA88k69SGZdQOGfm6B5VkiwAK4PbOb2ZQ4qsGvJ1wksjwNplUlIJSFCQEmjS+1U/zojApYudPaz8SowUCX1BpWq+39t/uRLjJ/rGuv6bLoh/c+Al3TEmWqg47CU1GnaxvgnNKfU5IzB2GSyJpTGMYJdaTXMePuaafBpJ+nWQ502+Ww+ubN4RfxoI1O3Levijezq6urrP7Y9LFD0ycD7O/1div2fu97QVvjAGVtqusPt9cJSTbb59eFcHhPrp+J4L2lOw2kFqjVBJiSXEEFSp3mp1d5CQ085oSuH5R105rMWIEPAERDUOUpq/0UvKF+fRNVxjmUWnmn+VCUIS0vT91Lb01a77BDAqYLwnerh+V4TsYcoDQoaSO+l5hetUFNCneadfROT2sVGptqoVFt/2+MfqBi7qmuo64qSEiHABj4T+yXbQpjYyMJ3eSGABaso5oXrPrOiqiPm9n31bkrUiSq51GlqUFIbzuaXLXJQTnsM+YZlVqcxIuiBCBmsV+6EdxwQwQ1ECHf14n2FSaEcKhv1DR5zr2rr7riVZySkH584O6xO403glHg4RdNs3OkqA9WMBSsRlvhuYk1hfmczXqjT/KuhTquHOu11aC4KsP9M1CLMVBVTdVB+TTbsBp7ppa7T9sf58XFZxNmNLFxQzdTUwr2/MQRefjnIwFa0xD9e1m1WGFSiVy4BHKvTXLBnzzZ76Xycz4Wqv4I6jREBeuWmxhZy/3FcrxwuIvDcMDKwlHolpNR3dlJXWd8D2ByTprKD3166U8KJ2MbMnWlt5gS/L8AabNnwhI8WilSBegyd1PFUH2mWsOFPmNQ8tO54IzMLqqHjI96hD91rWrvav823Jm5ksg0g1IeFT3khELTBFvx2e8Nl9Tbz1cctbn+sKqD+2CEtSt2A+IvsWFhX5DrJXAcBHCgdFEBRqn1SNYygjAgp8E4zjuuVWZ0GpJ3tbD6pJumnoIT5OBSA9nedyfUNTSV8EchwVgIP5FeyMKvOU7lW2xL/r8OnIBkwyGjPztQccg73KqjTjrWQWAOpeywGDfhGVBhOTNULlcrm72Mjq+h/EGNJnLiRncPHR1T5wsN/cwhgMsFlCpLQqKZDv+dAk46StS4YiWHmZSjsnTYMln1DtpvWu8DNNkKdhnO6bAK4kF55ywjVH28mEXpsAVZiswqpphm76IJgbr1K1fYjK8VuUH1nVOz/z1gh6TQjOODIojZpx9LUJ+421SfQqBbUPEDTGqFM00x4l3mKoE5TLIE6zdNKvTuH4bCCI0mk1HxCa4JaAbsBr3fwgDOlYXHzg7h1G5VK3EhggaJPANTf4VeJGrV2Nn+nrj9zbeuA2wt1mmybN5theMErXJbqJuMZeKfBMEZOdRo5QOEWAxH87dT7wDCpWa88R2ouzS1TPrcoKGOVPtsro0JLee8P+Tqr21BUaC1Q11d3TVJ7+uWjLaI/oAsA8+RZFZIWATg4shhahD+3Q9IHll2cX2OSum2dStX6hBVuuvZboW5bKYBD4Y1sAdHlmUfZa8XkjXunHTPVdehL3q4LUCrs2X0yCY2UwPJ+6LGLYc++zAZ1WpcaFnBYmPNbm9PCKYQIo0v6qf4vbUAJ7GmsTptvAf+jWqZQ9bzS7xsoc37aOmi9roRK2KQoUCKSStsc97iiAqRdTqccHge0COJKDzV3N9PIr9ykhBaB4LAyr8JcAtg52A34+v44pmitsEp28LvQ1AKizwuyf7uXYQElzV1tS+rOM31pJo8X3mmK4DWe72j/+aFOi4/30w6jj9Sn4J3G9uwyHREkhB7FCFZCr9zbQsO/cJFqPdRpYL3nXVCtJJhbqhJanugi60n7bmyaBqbmN1lavp/akbbcNQB1mjpM67lIO8Tt47SjNCrIntpNzc91kSYFnMo8WPaJXZDsBopUyp7f2f39J0Y/2zncfVUJxiYbmzex8YXv0YUAs2J8NneP1q7543HDd9+q91A61GkOUCtG8GgX8LTU6xHopgIHZTTGkHcIF9g4xjd+qEWD3O509Hbivam+T7wmVQQdssqgJHu6lZpxnlYnAxFYncYPRqOwADFWUI6dHPOOHPMuaVvc8a0DYskz6gOxD7o+xDj0UKdhvLIUbM/CiJJ82xzUUImoOPuhDdgAOMLXPCrjQ7cDzoCgWaSitp/3UuwyLdsN/GkB0WWZTXkr3bUriFP1Vv3jzcPJVBA35tMioFhgOkybR3dYAOfFBpKe5KOtahVpu2PJl401KRcziPYEUG43Ys7Vn2kiz1uIqlIERJ/J1DXS8aENSUq9Rq1subuTFIt8t+XTP18aZ0mO8ST4/KIGiB4mLPkxVBd2kah5MsJDCe1k3dNP2uUQA0RzbNwT7MFCnKAcrRjz9R9yrGhqabl7AdHDnqJPxoPByJ+C19LddGWHPf6KfKPdvzwR7haTBDm8+EKLcOLndKMILdbJ65v15ozoBVk+stm7qbLgDGk1cOBCe6F3pqtzuuuhNqZ7PyAGKMkUR0O/cpDSBGo3HnRhuvrmel3QCAoXuQODtWNZyy+IyRpJGghQmoz27DxgcArCEi/1lvaRvxsqsVzMUz+uR/MQjXYYZmpQ9d4/DVDKJbHXLyD6XFfJ3+A9sOysTpNCELR26x//dVkcDTvdiA0qskA8WEJfeFHx98nYFLo//vjZj4nPjT+jwuLrZQFcdgAS4cNoa4hu/vNzoBhQbsvkkk0KLMkBK33n4uvpuhsvppo/d5A2FwQ22lQP4w0MIpjkarUiYalBbDzWElg6vEopnIE2zQgAgCrOXBhgE4E5+fektxmRoUVQwDut0VtNMUk6Mn4Wgbff85AiGzdnbW9SfTP9RLcESB08tV4h865kSlpqfGwB0WcC2Cfs3rilk7fN2vS9v1SlreoZcftS9AGVXyYbc6A3peuVtCqjkz73xnP02JrL6ecXf55u/cPzVJiaRf0+38f2kfmCjFEnLi2Vnjr6Pl144xpKuyWJen7ZH/RUi8QabqaOMCIYBXKXeWnJU+nk7PYITQ91KpNfNlOyJZtgDUdwAZ2pBtwbR3L+kB4d/z3VW7jFW7ES4aFGltup8qdNlLwqgbJvNFM1VGs6wDhqZ3RuH3uVryrgS7kuSWXaaHg725T9SjQZhqmGuHAtShBgdVrQ0ulwSk177IP76n0IDyUqONgjL7Fo/ymhAuqFQcf2fA/VdR6EpYxAj1Z+SKkpZvr04gKqHhuiUZChbkSCiOYfwrGRDUJvEvT0wulSSrzMENQxRwmOUjUgb/6GABk/p6fYrBjq/I2dhAElNUAm4FvtkGzPJR4pHKByhbM9xxZwiM8eWOWhpo5m8rwRoN7dg6QCgiffEkfeMmyY2igNEH3BJiX63X5V8nWxYmZRyp1c8wKiRwm+clcTUqfVNGU+crInJTEQ8Mtmg80Eagy1r0hUUYK+nu4oP0QJ8SYaGx2mYy0N9IUt28CGDlO8Iiizmm2dR3p/AHa1uXFx9KfycqoJNFPWQ8nkPuMF7qNnXNl8CiOCToAjjp+ybzCTrWyQPCcR0HKtmjof7qcudQcpstAAYqpHi13hqDgKJJmwm7qp5bnuYHgoNGHdZ6eMSxEuhwsDPRpjA4y8FT5f3h2ZlLE5+d+TYPl34tkTiLC3UD7xEGB1GttgO4bqNtdY4795sMUlxmsQh3S+C2OKkXOVbBwzjDhw5+cN05GG/dLZUodUX6mxcfTj0x/CqkRLt2y4iDqH+8kkR84xLPpuP7AjPoUeOvo2XFIFil2nJ/8gDrJnLdOn6HwYl5hyest9ZP5uHKl0Sup6GPHg14LEs5SjX0H177UihPQQKeC4M0HyEUbNUz8iMjXnJBOrx6j+FDiGUvxGVBzVeqRZenaEvKM+yngoidynQdVjGNvnUfA63FthzqdWJxbrbEsX5+/i2jbcvIGDYS+UTzoEIPmW7N2qLIYnDlsSKEElypZthcND9cNC64IMsJr+U/Rk5Wkyx8ZSN9zhkMgM5z81vVZZRjvWFMHETE2eSdL+aMHSjXqz9VrqbrXSEftpyvyWibytMEefJ1Xn7kLuTek7TGR5s0/6ztBl33b1Wuju94xR+1grCcuxqbAP/EdizsiHhtfZ2k2RTdSl6qCu+8eTTMA4hu0C1Knwkf+tjZLXxkscRWCEFfyRNyO9weOCAM5d7QvkP5lB6UWmXRDc9rOWBp8Ltu5zBOtf7bVgAAEpPNQX6u3p2yu75c22wvyjAm5eqzNs9KfT74CCIuBjcJ+hIWBJjj6WXm2qpn6ng3ZdcAmNDNooXTlPMjsFNHm9d0DYF2dKpZJD79BYnoOSr8V5FtSYNFO8MNslRgRsEp5KUNBHQUGHvNT/6xFSb8Txg01P0SAnNOT4Eo1/sNBQbh8poXbAa3MuzBEoPJxkAtlWSltJgEMQJ5mQKsUwlFkIHfWOi4Zaxij7W8nkbZrHRoYpCPSKvoTzjKqE83Sn8tJznuaOb9iwAS0tnNEZBp/Ywuq0YACBN7W1Fv2je+sUsmdbsSEJQnEehEWDR+l/my2UqdPSwASq3cUsdWwCPVX+ARXmL6ElyRnU5fWQhhXuUSwhBIthHR/Y2zcbP6CU6xKAI2B9GdFDD4TbJiMCq9OK1GRaFUcd/2MDRYUhzsRwTZDqK4uAfM/DBh1Cs8BqjyREm5ONPYMDHIFQ4Ke2kVbq/yGSTIBdl5JMcJ95Y4HKUFugpo5H+8iQoSfjNTie1ICTiNRfjhkBVqdZvELOt8xMze/lJrB+OFyWBKkF1p0h8sktEqls6Si4o6Y/Pd825vLCHDX65HN8/F4c+lP0KspN6qBflR8giokn+6R0wCwBzNJoqLrPSrU9nXTr1ouRtriPzBDMyVG6/X5KT0yiZz88TNa4Xkr/rolc5eOCuXAbxFKXqHmrj7K/kUyDTWM0dsAlUdRz9NeMfGCrNTkqav11D/WZuiQhGgvTIi2SUw5EYEO5Nmp8DUkmwCuIHu7IuTVJgkGbn3qPD1DWtcnkc42r9iT0PPfZ6X6x3MF3wu/N+EKKMmG94Q3ExHsTjjqS01PonQVED0HiE/YppeiFPbsofpiNbCv3vA11WiqikHImBjmKFOwRkuYdi1xUbSml47ZRylQrabIfCa9TC5BPF5dED5wElTUl09XL11Hn2AglyGRE42TyjSPEb0+VUtwlMeDcgTQsiQwXFuAAfOV+iv/HGDKk68gCCjqt6SkIqgBrPN/7IjVUwBZ9zZgkTIuIqofUaaEkEy8g5wMEimezrUyYQMnBZp2Kuh8dIKVGQeZb48hTAcGcHg+FMz5+RgN1GgXUSVfrKXOd+R6uvlAsPGdLWUB0hsonslwv9aqqIe3hit4UgyLg43CBsswXr4gxUK1VSUoyamvp7rIjpDfGUeckas4dCq29BJa2u110qKmWrtm0Fd8dpI0y+y4BAP8MQhCYCYHgm5WVVOVtpOxHU8h1Bmw1R2MJdSj08ORP3BfABgERKOsaM/UcGyQfQuRMK9TjKjkePHzfrT8YICtSFityWHiGGwyoMIqkToPWzJaAJBO/7Amq06bzo+c6eXPBf51/sVP69iRMMmwjmHUKoz2JUyn3+hbfnU1pG0w/gTqtUoo6JNyAWj8qsiycj6pf+DYXCITCQw0P126t6Yr/yqFWt5iglS/YowLHOAdS9J6XO0QHayGA86igI5+55yyFTzPG01OVJ0jEOfq7m7ZTz1AfpckgmOOe9PGmYzTRw0ffQQomkeI2xJC/H9dmOjEwkkNlxRQy5fZ4EqBS6NmNPONQp80awAIx1dnrt25/KzkgTOMUxsyOz1Ykddp4kon6ciSZOIE+JAFjz0G7c2uRIt9sUEnCQdeAlzIeSyJPNag6zt2zNYlwQgENadQJO3Q9y5YseohrPnjwoKSlmdjKLNM58dGF738tCBQX75CWRXWbcc+hjriz6rQwNviIu8jeaQOsTssEUfaU088qqyTqyYg8W5HUbRotvXqmnLauWouUJjFkA1vP/Yx2X1ndlq7VUG9nL73fc5IybzEH1W2se54O//gWzsVMIdO2JlHnG30S8kpKhFk6yEIzVZGSBn7ikFIXcwpjtlWfcWBc53iSiU5OMnHPeJKJiQK/qYAKLGTZgBp+6SwkTCqMI915akjRMQfTbWShTazK51/8syxKW5f0AARvgyyAQ8rsj03eAqJPBfi/4bVQtpWunsYv1/Wlbq2zQZ2GFL3TreX5dpW90HSgwivSeujlir0478WTg30pwyhYVZSlN9DrbXXUOzJMj154KahsN2UpQTHDeD/SR1jin5yUTA8f2kfDWcNk/ny8ZD46pZSaEUHSK3spcw8EeDAOGHh+VEJeSZ02W+Pjm4QGlTe+ZJFSGLOtOhBp2jclARw01pxtpf6dCdlWpn1jwg3sI6xucxz00GDDKGV/3Qwp+rgRzVRNYgMI2AL+hOI4dfxm7fHctOxfoTbgejCG4ISapa8LiD4ZIn/D36xOC9qzv6Sv6TA8vK9BoBRdABK5WcjPHPvMFnB9oEDb8vzUaT9Cf2ztplwYqUxUp81WtY3DwMbG048hmFuak0er0/Opw+OSzuty9FoCBc7Lf6w7TOZrWd2G1T2Vui2ICKTfoqGkAiN1PIdQypnYgCJxeWVjl40COV/1SymMOeQ0x2CfUjAnWcBhxKD8rQMtNPjvSDIBjiCsJBMMZN5YcI7XFqio80E76c1aSvg8bBjKsOlOthvAoBV6QcRGIGR+JYmyNqb8gKvAkW86+g9OZqF8YiAQisPd1LbxnkpbWs6w0+vFsXLayZtPxxkJ3VhDWYjkkhnfSv9eXgqETaQeUM1IENQNVEtXaagVKrYqawf9C6vbBnpxVmeqPhUpmk+vCeq+AGUnJND/OXaEOvRWyrzDRO6T41LqUNVoVtIrd/goB5RxoG6EnIfgDpqC5R6JqgyPs55dm6tClhcb2eKtnHd9SkEZM0EKnYIGM5Fk4mVkW5lGnRbq4pSf2DMFLQSHzgD1fAB121XJONpD3abhXWDCG0B8T7nfn/n1VEXKxrgXzWrzwQMlB1RsczHhqXO+LiD6OeD42/0ANR/3TjuZV9NpuOPdBg8lav2yZFvhNcPeaXZYal2EYI+n20uptt8F/XhQnTZxTYUDkW4IyozGBNp16hglJSbSP67cRG2jQ1C3ybJHUT8L5nQJ9J8n9pFhm05CqsDEFEasTjvhp4QvGCTK2HmfnTSFYQjgphosmmJ1W6CCqL6shTwrR0lwYFxBO5TgG0zNpWwryNne1EKuV31QpwG1YIATUWF8BsehgSlu7w+HsHOAo/t+PJB6wkbGmxizYhAhmK6IceevzLmP27AV2mactgVEj2gm5Ht4VzBvN1XWJyLbSopeLbAJmjz6KrZnH8YiXGdWkF5dQw9UHCMNJOjWcUFapKPkFaZhPgAOdQfqq+nqzVvBLjhJL4fDC9oawzk5J9ZA71XX0mlnHWX/EOdZllKzHTxTRfiSS+q0q83UfXiAeBMQ5hp5gZGPkyOsVFHPvUPUKULdlg9eBS68IdYnlGTCZuyk9meRsz1vjpvKOOAl7gCVs/Aw/cJxdRs2HKbqkhbhlNeX/0AmpW9M/qFO0DWzXOeGG85Vp41XdfZjAdHPguJv94XVaSVguwYHay+u6Un+/LF2V8CoIYSHin6fuEr2YHf6lbQpZ5AO1OyDIYeaErASWFQ71ybtCP+UERNHz9aUS3Yht2+5mLpgB58qg8UcUIw6fV6ixFS67cjb8POGMcw2nGe7IPGHbt0DVj7tzgQMSKSeJwc5aQMo5TxgyUCBOJTVbQ372mhs6QApIHTjowmr0xTItuJBtpX6Ey0UOAn4wjsNO83cCzZhPt8PPj9GDpubsp4wSTb6PDZsOn5dml6deJGufXFu7mPcSMiefaYGFxB9Juj8Fe6xAC50toI67YelLQZK0sJxjCONy1A4omsfFv22LGiLnGX0dE0tpRoM1BuGOm227gwB2Umrp/899SFtKVwDE9o41ItcabO9GOF9xjvGo0wNjMK7B+hA5wlKh1DK34320ZgySUGp5ydSx2s2uNwCIacwPY2oScBMUrfBVn3w505kdG2G6RkYLla3QWAnJZnwtVPP/UPzTzLBHZPag7oNIbQ6/stGiRAm6rchjTLYek+DT8y/Pw3U3HwfJOzOce+0WffnaM9BRPBbeJioFOuS4dBuafqXuv6MohYp24q86rRYRHMtSOuiFytAzTVGcjKCRqEwS52t09M7HY1kGeqnPVC3iQM2yoBgjhcac8/R/GN1W5ophZ44/C5SDg9R6jcSabTMSdkPJNNYD7zCXoYjCSjjVKanEQ+XkQ8bhjpGhYyuVimzK2d4ZUOaUJIJzrYypUQ+4sbwAnYyBTYs1zF42VWNUM5XzeSs9vqSr0pSJW7Sv5+dlPE8npK0NOFUv4Do4UBJpmdg2BBMdyweSaq2xDz0enWATAg17EK8dM64Eu0/HkYXzpYXL/JRW+8hetNioywYoQzPoBuOdOidOOcT7OBvP3GI8rOyaXvOUrLADh4RccgHxIzmXwD1dQdAWWFs8lJVKcXDDt64TE/xSwzITGojDSdknM1YJZIB4rTAKY05o2tjYxP5l4OcI3daS1+zlGSCPd/mdUSY2BfeWFg2sFpFlnv6SJ+kFeMv1yvNV8dS1ubUu/jRmdRpE6vi73MVUUyuZ+H3HCCwfft23mgDDa3pD7SMpCXHKl2eeA0ymp1DYDHj5xTm0iZfm/hAiIub9AwEtVxvYaKSkvR1dPuR9yX9dw9LsKNYuOtMwa3Ddipva6YbN28NHNz7J2FbQrLgAmKyTfdHkoCJfQyNa2L/Q9e4g5Ov8zWWNoikj0+il5pb6TOLuqjomQJqfKcTZ1q/PPHgoetWI7UxpzjOeqKTdKkaat7TRWozywHQx4lD4i7Op6CukJ9Q+59tvo33Fqr9Me7fmATTB+JLSOKBqEPhVh/NboXb5sJzgADO5pxthdMdF9ZaYiot/VpBDfEb5EdRF7UzirCpqxOs54pMJR1v2EffeuuPtAhsbzOQL5qFFxS3l4RR9DvHxKPfuE3o7IHL59CgqFbBRy6K3MPZfqPRAIJGaOBiuyI+n05vQebVFQCkHNFxGZfjYURz3EOFz2eTzqShss80kuE8CFb6cTPaPDJ2TkSsDXjL/YqNB5Y4N+xYWiAI+nZWx2L9nEMSzsJjii8LFH0KoPw1LoWyrbTbdI++1ZAh1Ha5vDEqUS2HpJ2RfAz+rctAzXWqMjLGGOnirEX0LqTiMWoNTF4ZNaNTuKZsUPSOIXvgwS0XK5akZz3W0Nr6lW/96YVMhT42ABY+2qgg7SzxKhUN9fXQnitvoKIfF1Ld7R2k36CNbhhlBhEwxt+KCLJX6EmjQ5YVCDaTv4b4+r9GsolVGBrY+6gV3lQgaXeXe/zLH85XxGcZdzOSHzggBZSIaIeOPtCjNsr/dyvCbqwqKRECdnvt5XU9pn843u4OxKkR0w+FkTL6fyLO+0p4pw3S88d/T0cam+lL6zaDxI9F3YdcC961w+P2rUnPVly6Yk15clLSfVtWrnnmgW2XUWCg158FKzrmR1kaHrU/6OslB5sEM911fD/5VsDYqNhIPguOJdEkZYx4sFzzIVAEx2TvRQTZ1pdh2//ZZKgmwUFIYtUorlusBZjF+mOyYtTxWzWtS5bkSWmed+yIXHm3gOhRnJdwqgKSg+MKnq1q2o27S1v0HJ8dnCdmVYYSCva4LVukgZFj9FqHjZ6qryAtgjx+fflasjpGyBQ6CEah/VQ2uRsbUty6/nxav3T597nKJbm5P9m6rLCRktPUFq/Hj4TLEnvPfGe0/ti7LQ2BMqhvmPa1H6cMqNt8PTP4nc9hrFIEWejoTTcbIX1Xku3xIRp6wUE+J6LgPJBInlPjRjvM1sy3hKh5nU9c9EAGpawz3YN142SbC3xG3MICos93QiJ8P2TP3tHZ+N0ae8ba9gGXT6uQT53mw1kgAbnSlqRY6LmT78B0NB49FunNqgoCxZWoq1dCuwgHMsXjCTiXtzvGfP+0aqNi46JlzxuNxlIsTB0v0E0FK+/52fbLEGK2G7bxyN2GNnm1RuuPt0kOO5WWYKIfHSmlPnM/pX8Dft3IxhKt5AihE07Gp0xk3W+XRqBOh2DuJRuZNySQehnGxamjooFVfERoRrYVqNMSNujfzUnKfLEE4aFCNhdTgH/GS9Ho0owNLNz8CAIsQAl6px0zVXUYdr6D8FBmzrYiEzUPhYfalocMIV3vU2nnAGXDnj0rJpZeaDhNw44xun/teTQ8MkjpUbBgA1MO7A2or1q+emztsuX38shtNhufWgWTyfT7zYsL9l1SWKRsd4z4YqPIRXA7IRLnkoidll6sPEiJV8QC56Dymu8qR+WS6SkiyKaXJJEX+an7fjlMamR5VaQjzdLrThppd1DWdxDJtTEKVJ33Ci3CQw0ElCk3GilnQ5rknbZT3Dlntm++IGAYL5QwIfAfZWXSKa62OXXXaVuayevzeUEEZZkDXhFYj7QsXkkmQwPdXvaepN/uAtXrZz2bIZ5+dfIorcnJpyxkYemCSSkQdc7aoRSck3vHhn33bbqINi8reAxUvCNkg80bHINoy8pVd920cSuiug6qEhUI4XIWPfludAqHncpBGKxXysuoXmilrJ1mcp+KMJjk5K6g9xxBlu3dk9fFU8eLfaSG1x/HZpfCTi1BJNef9JExJ4ZiP6cjfy1kA5FGcp3QJnMgMOP1ZX8vTUjflPSLBCGhnLOtAKZz1oXKssgm9Hnh6zgEsNiV39q40esZrVyLYI/fPtjkIg4PJYeUnakbOzgNSdlWRulY436I3WEyigM7i2pZyp6NbCtlvRZq6rbSHeu3MPJRKgJQ8LuRFt69ev3YQYwJ6m2LljfmZmY/wXWEbLB5gUpnS42mfMPyFc/evPVT1DHc7zNx5lQZSg8b0RiT6Ymje0lznoIMBToKjGFzmws9BEAkl9cWRJD9l2Qabhuj0becpCzAqJlXQbUKJGz04r7t5CBlX2/GmX0K19IIxonpgehUqTZdGtO/dOmiB/lVzrYSQRUfe3QB0T8GEnku3PBysN7Kjvjdx7qSKVYFQbhM9uwsgBuAY8TmNARREs/Q7jMVlDQe7JHXOv+FLNjuQcCI9EQTXZa1mDpcDoqbgxY/jdn+0UHxp5suhACugG2wfZNtsHfs2AGUQADKpctLLi9cNwzVAovOpGt8PZollOWlqbmDPuyvpEyw1LARl9jviHcyUGbfKexhVwUTMlp+BpdXUPCzFncAZsiCrWvXgJTmyXSLkbzsI6+LcFS8qXCMu0pfYNHuTEorMj0CWPaNw3JesFpA9AjnYi6Pg5qrXr5B8Pf1NVxd22u+/EyX22eQMTyUdFQWVbQuqw+RU/ehyzHAqyCt5n/5j1dNGruRInrr0aY6unHtJsQ7A6LztQiwIRYbQ6fH5duWs0S1KW/pgRST6SVUIMki8Hm2YMEiapWk/+0uWrHi4V0XX0m9/d3+dOjco114I+MsL0Zkebnv0F5yL3ZR0pVAPiB7RCw1gCTFXfdBnfaPwYSMvmb4mnP02Yloh5+8P3K7LKTLuMT0EQSDYA9viLzzDQT8xrUGVcL5+uq89Oyf8oshzii8SqZ+agHRp4ZL1K5icWONc3z265U1HTF79jdqoE6TLzwUe6f1IrDhjtwA2Yc/pOcaWyg9Ro8oqhNXZnB4HAAyGWf1n1SdIDUQ7uYVRWQZGaKUCARz8QLUac5R1VdWbaBNS5ffzTWDTZ9uXUnsZ25GNtRtK2tiUzLVkA3AYTa6JYRbBs7yMuSmv7QcpdQbEykwAl03CyJCD8zSrBCDszJSGpu/HU9KRI/pRkJGNWzPOZDj5GOAFMkV7rB2TpyI2NkZu8YTJ/KmEE57XCVT81Y/5Xw7hdLWJ96LdYPl81G2lVm6O+Pt6SZkxpcWbkYEAWkdt7bvvqO6L2N594hb9mwryToVLUpmddq7kjptZAbvNB+vQiD565XlVFywGt+V5A3pkWYZphnUvxMCuH9eu5k2Ll7+n5qYmA9LoOedTgXEC/dAcOH6i5YV3LVn+6elsFNyUHXuOqvbMhBM8qkPDlJPInLE/SuQTwo7xbR3lsLIyciHj4yLk8i6145vM7zH+AwTYxWsDzt+1yclTtQsh7ptCBtsGFjGGxDiw3nTbkhWJm2MfTMzLvOPrE5jIjFLT8O6HUYXwqpn4aEpIIDdWJooUazIrOmMvf8vdYgvjmwrMMuWpbA6rRfBHosXeaiu8yDCIg/BZ1tFozMgLkdyzdAZ6KWWWupH/vOS9RfQANIhZ85A1UPLHVa1AWwM6s8sWTm0ZtnyB3hQO8fP4tMNsBgL9/qXrlcmJSW9vnnZitevXLVRaXGO+oxRVreF2ncxjy0gy8tpeLd9Kgacu1Kyiw/dn/KTEZwjyCKVceYjiEk37KP+/xxFAAvs2XBYm7ZAOKfMQxTYt1w0zIkTb0Xkmyac1WeLz87rQS1gOxbVpn+Iofz1Zokzmo86bXIfFxB9MkSi+Pvll4MkoKoh8ZEKW0qs8FfItlKYpKQEXR39oPwwzpLxcOOcXVgr5VczGOkX5UdpVVYeLUlMkSK4THd65nWZCcqPDcFfsnkbFS1ezkKj7nCFRi9dz8d4oo0r19x14/otPsgGVIY5WHtJlczyTz+oek6ckd44VUE1/mbKehjqtkqo22ZCPmAFU2IpIePqYEJGFQxjZvVOY6rOudtAyS1P9lFspp7irkWiiTOYgxnUbVK2lZNeX94PMiljo+lnsULSmXELuNknb5bxh24vIHoIElH+BDVX3gAB3PBwzfm1PYlfLW2WP9vKKIIgXJA3TIfqIYBzIY8XZjeclcJEKgM26JX2bqqxttP3NlwAddzQlFlXmJrzWaTT6/VrTSnqC3KX1uZnZ/+EwReu0Aibgn98U6jZWLjyZ7ci/XI31G2yJGqElAzhz6FuM9NjyPKi3ICva5G1dLosL0zN+azcjA0CmUkHG0dp7F03KWEYExYw+X1I232dSJx4DAZK10Hd5oF6BTbyU57V8fzZbCvFOhvUaQ8xLHfMwhnxM5GUBUSPBFoRPIv1JUm/atvidh/uMFKSOuDn8FAhtjeCqmZ9lJ1gBqFOOz8denJfBf3ozBlKR66ynikEcNNVZgUyxMclInHiEUqOS6B/yF1OFkjhjZN6HKTmoPUjA/RjGMcULVnGNtj+SIVG2BSkPaggf/FDl65c1006vRqyBMmxNNowcuJ4wumfOzu66IjtFGVBH85CrympOqvTTgMWN8ZQTKqWLHcjISNHkI0ggAWHndKsU1P344gtB+mo+d+QaIIjuU5Wt/GmAGGdp8rrX/zjLEpfn7QLsOwfh6W0fqabr0ivLyB6pBAL43mmVphCsbu7/gv19tRt1T3ItqKGcUwY787lESwMUsF1alVGL716ai/YRA4PFQ4tP7c1Kcc5LOTer6uia9eA9HncBJb6nIf4d7vL4f3UkhXKDbmL9iYnJ/+BSqC9i1BohOcD41R9GOq2XY9tv5xc9u5AJgSDvJlEs/AIOOxUXJKZSg7vp9HcMTJfEych3zksNSMeR5BFaJ+sq5Op58gAAttid56B7Z6yn2gQVBoyOAgr30Qk1+1JOMPBkWfyAgD2BQZEX/ymOHXcJs3pvPScp8fri3zypuzIRxcXEP0jWETlG5BuPNtKiaam0/jwvkZFMNsKaNW5KBOV5mABB4ctZC7ZnuunnoGj9HJzJ+Ug2woL2SItNnAAmVC3PV2NxIl4/7uFG6h7DKmPJN16sDZ4urE6Tf0l3CtatPQevirunJuxfojVz0rN+OWWpQUVi7IXqSzQyXNW1mjCiiHBOKblcTj89FbjB5R6A9RtHOVDx1jJowiy3FJCxtviSQE2qfsJ5E5bD+MYUOiICzgs1QYl9T83Sm5YL0mJE2Erz8cCqT3eVCAn8LR4hZx/TSHzGpMEy/GzeeSTN0sHFxB9FgBFejvkndbS8eUfVPelLbaPeWRTpzEyeLBY0/RKyk5qhzBtP2zYE2GOOneNjKSK0+jp96dP0NalK8AdYNPABsBtJQNR2qFO+5cNW2nT4mW/0BgM5fMRGoGqS+o2hvGGgpV337n1Ela3KVKZqs9ho+J6Zip8Vk+PT6BfIstLp7GHMv8NEvUypF/mePAoUrpjjDTtoiSy/LkPsgg4xExMDDFT5ZPvMT5z2Kl0ZHn5jY1MK5E4cRMSJ9qx5bCQg48IUKdl/FOKMnGd/k/Z8elvQkipnE41Obn6SH8vIHqkEJvh+ZdeekkZ9E77MLu6w3D3vgYvmbQB2dRpbM/ej2wr2/JdVNleShU2h+SdhqAncy4cKDKHEye2InHi0CA9Cr9y77h3G6x+AqRSqT+9pLC/cMnSndzIjnkKjYqZ5UeQzPj4+LeLlhS8du36CxTtI0PeBEj15SijzD+rY+m58nfJeLEe+AaKDX2nJIBDEojM3UjIaPfQwP8gISMocsTZViZ2Ggy4MoMTJyLBIydO/AbUbe2QDYCLwBEhqE67Ksafsy5DCvZI1098ObrfFxA9ivA0m6+XSENlY+qj5T0psUoRqUvCMpeIvBMsgBtB7atNCorV1NB9EKJpIESzQJ0UpE+R18lv8LvdzBEgxdKT5UdoWXoWFZnTkcXFR4OjQ/5HzttBG5cu3wlqbAtXnTZbT8SdQffLTatW33Pdus0+8jjUhvGcR/MZy1TtjmAjyzbE0F6kh670NFD27qC6jYVt+gs1Ugz1jl/bJErMFHm+wBThn65doSbLHYjkmoLEiV+Euq0xQN4Kny//PmRb2ZD8ZJwQV8uwnC3bylTjCffaAqKHC6lZnmMWtrhY8A33V11UY0246SiHh9LI453GXeEYLa7xbCsHa6FO88C0lrFfuhfcXXhyI/3jGnC8pGzEkmtHTLkzHa30TfZuG7L716Vmqi9ctOxkdnrmU3gkbHUaPztTwabhPxC0mKvbsHzlj+7YcQV1Dvb6OFEjl0jHMNvzUuRbhJ269fDbJKwNUPzFBnLWeygbsdM5IaPjSATqtJkGxvd4r4BqLYD/epAeKuvKZPKN+APaPK06cZvesmRR3sP8WEhewd/lKAyTv1rhOOYYOdbSX+eP28NZL9pEYUp4FRfvkCSl1Zb4Jw62xVGcSgx4QVzZDTXafzwgGyzgLshE2Dd3Of28qpLikG2FqS6fbTnn+Xz+UInkFELGRHoInEKiwShemLVI+MrK9bRu8TLJautZUCAgKMOC/5n3X/GuXZJMevmiRY/tKFjdhcyuar9X9CeJMACC02YCUiB99MnfJ/9Nfi70Tui90H0lGfwCLYbsgSyD9H7XSUq9LpHiPq0nPSK6dt5llyhwJOo0BsK0BZCRfNYh1OP0UNBPcCgqMfuWVMrcnHI/YDg2zhlhpchXpJmSr/qPamaEQ5F1MB+1du43udtG/RzHy9fV1fS1/fW5//0qkuCl6AMqaFim5fwYEJOBP/na5N88quA1puYqun5VN71w7Bl6sWeMViEiqScCvfm5EPr4L26HkySeRgDJb+Yu9V+35SJlit7wQtGqNV/6+NPRvTLaP/Slp95+9fm79/4CCu1s2KyycDEErRBU+JMLX594beJzofv8GSoTnuXMi75eev5TP6Dl6nxq/jMysDwDU9e1UzuuhGqY0yds2f1nAmS4Xutf8bV8pWAUDxZdtHLHnOqaw0shqMzh1fBfCSFCY0frNb39ml0uD41AfSEnNwHTC1Jmp3hcCTHCF8zmHCv6EFEc7HBHF9pERLEq9mSNsc4yGJfh98MUitWoMhQmoh6s+5Q4kbyKNmrsg0EHYrChHzK0xgRIIJfPK27JWyIMul01g06HHfHZ1QBwsEHuELcdWkkhPJJ6wxdDF4KPT9lJ6ZFgBRzYXqNTCpn++I2jrW6lFMV1hlenrC/Mi9x1nytAcbmICuPx0pELqkm3ATnO4OV2djxh1jXrY+BXFEkCjR5z+4pLi1TpFyZ8NVmV/FxVVZVm5cqVfFqStQQPQTI2MY4IENiWaA6Wa37ySl16rgN6SRUEmnKsTSXQyw6x80VZHlKrIe0c7CvB8G4uLZUQT2IPozxcjIR8je2x9x5uz854r8HlTUCIVVjBhVHCRASpJjwLhojrjYFE+oq17fTD9gqy+pTEqlneWaJddNiLT3jddIcpTYixdYtfaatdsRES+TGoqRDSImothqDAxjjHYaTzzKIltLFhFSkrEZcdrqLhpymIDAIMMcmtV9sFOFoo7XaoJn88BKs2bJzzFcRN6oqgR0CaY35K/rJRcI24qPVQ17/ikecYyUPEYtIrUf0pO6KXlUl7sre5/Sv3nO5Nz23udbkREFHlkWkP43xlRlg3LYrrE699clB44tqYb4pi+6/AWh8PCsyKmQ+MSmF1GrPsoqt66Z+O63/wdp2HkhEeyoncabx4wyuhJ0OfU7/Fi1KNTazTqaCvrII6bbSK3hvz0mqwhGOggjO/PXWds131sHE3Gt6MjC5/6GyFfa0/4HQ4RFYlRLuw8Vkb1FzFcSq61LmMhDMapUvrJYVXjpFhWAhZJwwqyHOBm2oReMP2uxFaflcW2X89EvQ3Z34sWnsn14NhsEAOwjhlw+87/OYlps2WfustWUkZvxy3vZADrGenSVZEx07F7DLcm0/mvXEk5q53Gz2IeorcYnxcP9uF6H1RwWirF4jwmWVOqqkfJduIwveXk0jTk+97Aq3AFuFgWHQ23B6NR4eiU63Gx8t6zSq9wsWTpR4XfodbTVjPMdswii1qo0mg2PgWuq/RSis0GkjI2fw1etSVO8Prkm3cy90u2o2MLn0uJ72A7KjrtToFu7xyX6KFA7wOuC4dMKFXdNNTKavIfCoF3qA4ezEg5UirhPY43psyVaAuo5W6/neAPEcDNGp1UtYDJmq5vSd6LDwGxxZw7pNeSr83kfzuAA0+NyY4TV4ybtTsAo68CGwYLAGulESQYomHEEmR5RwZ6sCu0lKp/soG0+Nl3SnIjMUKWllwnIWZNOwRaK0Z6XKEIXrmfT9tzRJUv6sd9ZXX63b02Fq/RFQihTMK9W8+n8wdvHzDDZw77dP1tuTrjra5A0ZkW5HL15zlmC54p23OH6JSexV8qhUS28lIEi2kC8GDd3/Wx18ENn0pEhg+Z22lDKjbOBtKCDFDz873U0JyLAl4adI3Uky0ZbgA5yAcDPS4IxOSiyAIAgiCe9UI1R9tIf9RmMeuQBqp/+ij+EWxZLgUZ/YW0IRokEEAjCX46gwlpWyGncNrfaRKViq8dq+3f78zpaGpZRfDcGfwaDlfcE77vmyIjp1KWVJc7BsZrL24pjvp80fbXdArE9IITtuXed1gmuaE2uS8nDE6UuGgZg/MF9HWxiSF8NS7Dqqo9T2OPsVIrPY8VW6oRwiZKtYiPNSBZh2z7FK2FUaEaBe2Z+/HJrYtM0AOdR39zDZA65DjXEpDFOXGeHr0QLxen4duysilmkE7HXM7EV5KIUWQjXJzwY2D+Sy1n74Uv4qMp0D1dNBLyqUV5QlC6mhhcYA6vG3UsxPx2dcoJWs1prr2M8OU/UUzefthl86msfNZr3hXyp1W76WsO5NptNNJw68igmy2AkEsVKrWJzup67j91gFxYC1nRn0JOBNt+IbqkwXRGRFQJL1yZbvxh6UtBpiCgggFTaZDbUftkxHBDr3ytmwfuRHz7IfH4bKZAJds9ECjFJXVQ27v0RpjdlNbm+Q4AMHcvAAasmdvbW/4dl1/xrrmfrdXpxJV81kTMwEDPqAUCyljQbaNXumqw9kALPs4dZ3pvbnc0wD1yuHBdlN8MsJR6+j+7g4qhM6ZQ0QzjkSzMLyw89IZr4dKYIFX2LWIvD1oByrueSHYDJ3k/DDsVTa6uJ8a3mwH0Q4eRCRd9yqke36gHxaGKjJ9A8EkES9uXllewBEwZxB7uY7i8pCz/ek+0iwOBrBA2ClBqVf4bL8fJUt5Dx8tqVrYJdcSwojlKRIitVuabqmzZxQxImiV8iECG4foIYArTBmhN494KAdiaJZO88JkZL8gUVDtfLefymu8d4liz2K2YMNmNKex83tBe/YjSXVdsTvfrgtQmi6gZCGgHIXt2dk7bUeuh9o9VfQagh2uRtBDOSQ3vMpYCs2J1K9Iz0YoKisQDmdleU5bEoqN4KyTE6ukqzSrSX3aQGIsqAE4M1kKs+xj4P0KPdTS00zD/+Ei5UZwfuPOAUGFr0hdBxBK67Lk4F4T3Aci7w6AyRwBcwbZXzBT36kh8sCWXmHEsmMSiAlUFSpVPa/YfYPlzk9bB6zXluBoyUfCyBub/Y05LfaZqmVrtCB7fCChrjNm11u18iICx0nrwXnr4nw3dVmH6fkGmG9ClcHGKlx4yQCuQpIm4N1/yqA+WTP2KF8PhXni75GUj7KtpN9f2Zdmdnk8XsgHog5H7lNoo8o3wNUVudOetrbSIo2WxmSgrtwes+ynoN66KyWDvED2f+/vwREheDbn+9EsPD0GYFajz033pi+m3KYspDoCyz4vXmvmHrKLvgKw7E/toebfdJHGBIeWCWq0UCTXvqeHzyZOdFeMu5bOXPXH74IrYY4g+eY40iCIf9fOftKumWCIg8nls7tmiVJo+VE3WU/18dFSCqzJHPHHK5zflagv0O3bd0p11rXk3lfVn5biBCJg/US9ndCw3dj9c2E8kmEYpt++56c1CZBwMmaHHsAnTFGp0KhQPXt8MHCmXnnjiLNjO4d5inT3xARI2VZEd9mKGovh1tJGNyVpAyq5BHAci30Qsoat+Q46NVxF1aA88YCkTMyDdBxIgvHN5uR0+kNXG+mhr+dkCBNhOQGs8/qqQaVNEMBdmhRDxd6VJNZAwWYA+sskgCOJmkPOsNpJjbXN5HknQIo8dGKishU/Oc2SKkFJlldslLIRiRNhmCeFd45ktEzNMU+8mXFCxs59HEEWv/nCxIKJVMQplKO1o97hY95lLdaO743fjvp2F1UEZESQ2GL36YIaS+xt++AoYNIFTYtdxl4AAEAASURBVEEnji9a35ma2yGk2p7vpKq6UTrYB9UT1stkeDLn6UZe4oIEMfC7I0Snqry7uQ8sUItk9wxlW6loMT9+oseMIxbHEZGHr5W0CGDvNiRDzmBspge7umkV1Gms3sJwoloYXnxWrgU1vzc9l7oco/QqZB0FcCqZiAfRapTb02HvH4Ry8BZzIZmrzeSHVUwk1gcR9UVCYCBVGjzz9BYEbkS2laWg5lOFh8KAlYsUNPxHF410wO33B/Buq5slmOTEzmBw7PLqRgrljJ1I3YyUyvZfjEgJGT8WQZb7Be82zRq1sumuDuou77sf6zFtXGAcVdyMamW7dgVx7FRz4mPQKwMRAtCho9sTARGl74wIIzCmKEoJkF4coh+9L9J5SYzQUzfAVDdZJ6jebGJ1m35LZ3fr1/nJkGBt6rc+uooJkLKt9PfXfaa2O/nqik5XwKARZdMi8HblgzNHUd4gvWurxk+VpEKUA5Z8KOQUTZ9CltVFcHX9pRVHBLDscgjgGKK8qZS5PXRzWjKdN7ic/K0gYDpMEO8AMhQRBEHhUZKrcJjqD7cQtUN9F4uGpmKNAGBm57VLoW77ORIn5saQ8bMIJtmKh8M5PQOjOHWyBuo6c1E8tb9oIxUi804bQRbtAUMUfgStGnjHFV9b0/gQg6B0V2lUcTNqlTEilJQIAbu98fLaXtM1x+GmaYSbplxsLa8KL44yG7NAyU84yYbvMyEB33Nit94IX86fHxij03W+h0TRZgwK1mYWzDHV512WJ6Cm3bjnQIsOLDuiMMioRbBxeKhMP40o6+gZ+xDOymrEgYs+JnCNHLrJBtf5z2fk0Sm7DX7aLik101R4wDCYb4F8D2xKgL4Ut5oMp+MpoJVRncYrHAI4YamfOpxt1PcIqCvUaVNS89DAABQpzBPSWtvKkDjxRkRyRdTYaSO5TnyPBXBInZz97WQaakVCxjehTstHJ2ZgjaQQ0UVQt/3cSr3lg9/sE/s2FZcU+yI9Woa6MdVnVBD9HETo0O8ubdFDnQa9MhBhJuSbqkPhXJP0yghnfGG2lxyDQ/TTE0QXxsPZY5aVyYua1W2No27v4SpDZl3z8P3cXkjANl3bIaoPLcJtVX3pqzuHPF6Yo3LeX1kKzhOUBMn6kqweermrnlRQp3mBHXLAUotaK6BO+yb8s2NBxXf1dNBqtU4WgR/Di+3ZT0Od9khmDq2wLCJfH5CK7V9lAibWJnJMgBXPs1PDG6xOY7KMxmYBpkTVYfPe9RASJyJUV/K3EUxytsSJGIe/CgkZr+GEjHqyPAaHo+U4IoDCz9ge+hJwi4JKp/T1whS384RtDy8sHC2n4U/5bmQlKogeQoS2zsbv1vZlrG0bcPl0yoBsemVWZcVDqrncNEx/OOShPKSt9UwSwE0FBp5bVrednyCoHnlvkCpq/LeLomUZpzPGgphSAAJnnHF12tFUZFt5cC/OazDjlS08lGTGi01sRz5UQK5qemPEI6nTZHINwJIPYtglaVm0H0guFQBqFjyYCryzXmMAszptjVFFn1OuJuVpPYkGTKacArgRBQVWeajZ2kyjv/aQsugjddqMHWYAoGv80fkXRHItHo/kOh3qMS5Dxcsx3LOuTabe44OwCQAXEAoGOWNjuMnqtpUKVe/rdv/AcUdxe3/n53FVZF/12V4N5/68ER0IMo4I+0x1FiBCvZ/TDil80dcQSONhAZwU9TTPRe3to/T7ZoGy4eEUUqfNNujQ/GVoAr63K2JUJ6u8j/M7ULdNWUpLg1qE2qb0hyp6UhIQnR1TIo8WgfvmAIu3HBtXsqmdnupqlwIkMMvO96JZGL1ZnVYF6vqAORPteuk/+nupCFRdriMCn80b/R66LX0ZZTdkkg923wILW2QqUrDHeIHsyd3U+lw3qdOg3nLNQl0n9IWjv6o3IHHif42QZ8RH6Q8nIQb7hEiuE56VvNOgijN/N46UGiRkfHSAtOzXPpXAb8J7Z78CDPysFimZ257spt6K/keAW5rxo+W8gTRvRAc1lyhhXcviB0/1piZ7fT7Z1Gk8Wgc8wxbHi5SiH6Zfv+ejtXDy4LN3JJBAqG7Kj1Uof10xFDhZr7h2cLTtU6xuA2DPEbcwlWctgmesZkNNV9zNB8azrchhxsuIp4QKaIDVaYvG6MRQNTWAjMdihqYjImcXyRy+8MSPQMiwDIi9Dt5pL1uhTlOpJbPaeS+KKfrDFncN0HNeazLSNlchBWoAakmdNsXD0bjE3mmjUKetcVBDZTN5D0AghwQXM52VP9YsAMHnZ3UiBHMvIJLrmjggLyK5DmABTeL/eFPhVZiGGO6df7HjG9riSY1kYTIHESsoHc1O78Bh5+KWjrYfcK0hjpm/z7XMa04ZEaQdx121ssoS+52DTS6YTQZUsiEC65Uhad+aB71yNSMDAiNivQA+ERWGvRsudNC5iy8cCtDpmqC6LSRwC1UmjGsRzrQZd39gSaB4VTDbSuh+ND9Z7sDZVraY2aijiR7u6aVV4wK4SNZKOH3i9cdn5Wb4mv8r1GkdSK74FuK3szqN2RU5CvOfI4KXvpZcSEmViJumklEAB4Cx8YsiC4kb1Bay7h6QpOCSPjxSYAKBWd029k4wcWLWzZzlZQJVZwaB47OD0mc8AorPCRnh6qouYu4hQkhyvzl32xqVsvkB2MGX22HJ6cgMR2A8W0vzQvSzeuXmuD3HrckKA9SuckqiByCJ3pQWILVviJ44ItLmxOCZO9K5Y6CAG0b0VFLutzh8ZXUxRdae9lv4euhMJFF3aBF6epquq+tLvfRUt9tnQLYVRhJ5CuK8QZ22Pm+A9vbWoIPjQiMZGmOkq4Qj4RWGOMqONdIzXW1Qp8lnz85HhApEcPm3tBQqsi8lXzvonQ6dkAmYIXWas3CI6t9rIaEXK0SDxubSHiMfW7BB7275MRInZsBu/ZoYjsmOOjEGUHaO1a5doybTKiRk/F9EkDVFdkSYPMVYm+DtRJ/9bYexvtb6CN8vHfcEnfxsuL/njOiMCC+D3R0YaLiqttf82VNdLr9BJZ8AjmcpgAVTlDlCpcdd5MH3+R7vWOe+OUlQPLl/lE7WeKBu60gK7p7jASXEZ9W1nfrH32lQUao2IPhkEhoxNWe5Q3G2nwaEWvqVfYTWq2GVFu4sRvAccz+sTvPgTH491Gnl9l6qxzndqIDQKoJ6wn2UFxgflVhPfkPsaoo5FUd+Pai5TLCUWGUHALrcT+2jrWTfPUaqdWzdFm6Pp3gO/VeAavsQk52FbNnXgSMBtFj4JsWDb4M6DfnchppGyfEOIsjmYNQSKz9FXeFcwsRr1iuVll/0BLpPDH7VLtovGDfumnRgCKey4DNzQnQguaRXhkRaVdka88S+RoQaxkTKJYBjxw4b7NmLc7000j9MT52CxZhxdnVaOGDAZqHo9Hi9750xmGubvCX8TmsrUmqgNLd/+o5qe/rSnhG3pE7ja3IUVqeZIcBZlNlFL1gbKAWZTUO+39Fuj6nrSSD2vySlkg6s+qO9nTgiBAVwQI+oFiagfESoxhHhifQ8WtaeBycPnJ2jIkeeuqtYm0hsCHVaro0aXuuAMg1IPlOUzqmrOfcqU3U+q8NWvfuxQVJAt5ryb0F1GxvSxF8fQ4Y0HXU+AYs7pEye0xFhYotoL+CCui1NGej6zSBZjvU8ybeBdHPei+eE6KXjoojm9puACBnLuxkRhMniiYk9n993VqeZ4KiyOGmYXj3kpWWQSvM17jwvzvn8MbW5MF5Q7XmvH1Q98F2Pp7UoPz8feFaeW9MZc+/eOh+lsnfaXNi+MIYtmfEi28p2ZFupd1TRO2M+ygKJZwHcfMY11btSd6RxgHuAOu0duKCypQMWUNTb4vaZs+3FRG2M19AVCsSAOwN1GnunyUXNx+3ZA6vc1NjWTI7fepEmGaskSrpJ0BusOSGYOHGbiRTIuOKz+ynrGjP1HB0gXw/GxkeSOaOjNEPBf7AAkOVF1f/OoHfwhPv89t7Or+EG9rFzBcYT3pjxKx8EIypoKOid5jia9WZF7P1v1kCloBWVsxmrRNTIhIdZr2wFNf/8Shc1NY/SH9oEOh8CK5a0R6s4MT+ZBsH7xxM69eJsD/sGX3q61vRQRW9KrBhweSFcVMshYOSjB7Q2VAgVUHxiK321oV06K/dD0CFHYep6BkEkdmXmESK60n8P2GgVfM6HZRCs8H4Sj/YsATc9nrqWMqrgEQdjB2Z3o4IIkwGEakUgtBKRVruTrNT6wx7S5IK6TvBOm/xKpL8R6YpURZw4cYxStnso5c4EcnZ4SIGNuWcPEjIiVXLU2uPxsLptpUrR9B0L6ZYKjwL3XsGmjKxZEkcdEemJGNGhbkYX4CRvSXuoqj/FoBOcXrVSxP+Rgm3257khNoRZbQqAtR2g/zjspa3J2FPBnhnmfFqZol30vSCW1L+rG6TNJ9WX2OxNP9l70vCPFRZMJuKzs297xICaopnJl/hsPuZV0Ma8ISobqQdkdZSpUUkeY5Ofne9vhiUfES6OjaPVSSn0f9obaLUeLCfO5jBDiHpRY5l0Q532pZRE2upcQYEWTFg8JpP3MBBZXi7SQppjy6HlFqqDjZgFcEaughFqKGumQBnmbAN8z6MZuhmNSVLxTCV1PN9Hi7+dTgq00fpiD4YEQ5z5HhGmgoVaULrJ4xk65E2vy268F4/cDcEcr/6ISF0ITlM18bFr2EkQg0DwI+/3ee39CUfbegURJqWciSSiej5W8TQXQBAInoyUhUB6I2N+6h+CAQMwDus1+oUnEeJa1mWvXqoTqtuxJoMxlGUbmw9WYnF6BaUl9tOJtkbS4WwekAzBo98k1+hFmGYzEN2o01NDbxeCdcjXHvAOKkwfrUrLpYy+dCko4rylpzPNOmCpZF9zrY0Ob66ElRlwISJUmKnyCfew9hTgwEaPu2nt83nI7qKlDz5TS8bzdBSA/IE3sagWbIxCnCB6ynzCpgNLfat25BbqhPgGjkAcSa62iAgVEE/iKTv69I+/Umkma7/Tr+UQSjIgHrO1o9CZr0z2kg4OEL+tSIKOHnHNZbC44+6rgeBdDoVwbcEYlVYK/rea9AqYukLSHtVpO1sZB3v0+NX0uRUWcribqezUGXr69BEqiDPRqJSZ5OyjUfnCZ8tRIPoiBHy88zPX0K/ffoM+HB2idAjk/HPSO03dLV4LcfBpr+nrp+8Vn0efdm0l9eFYUsUyNY/+Bib1AnOncKloZJONhlTDlPGgiSwPwc68KAw786mHMf1V3j9sfko830AjcQ4aUoxQ2rUJNPAmsq8iTFS0NxfW0SOAhWD6bpxXJ8ao68u6HkHnbqRpLDmn63jYiM765Y0bBW9Xb8MN+6vTisutbn9WjKhywVJNjsKIzsEe12S66MPWGMn91OEB9yBDY2DwaAht5cT4yGTw0u9OxSljwKk4sNHIsIfBSQWGHPCougoeVd39x2hPxXH6yfbP0NM1J6hWsrCVo1UINIGAJ6wtVNXeQl88fxsdevm/yWHOIJsPSBi1aYTJKZ+3kkS6fskVpP8Q0mklDExwRJEFmFztMISJK/xkGe2gmi920JrX86lvyxD5LRC+QHAbtUWDaZHCQ9X4acn9yfSG9X1qGeil733+eup/dZQESB9FtjiKGiy5PmaXYeb9qWTV8furA3lfS7+h29n9bJo+7V1w2FIqsHBQIixGgw//Qf1yiaauI/bR/Y0qStMjHD2oKyNktP+YurJxzMW5LnLCJPR4r5oStczDyNQeNuJRbFjbFjuotltHQwhmwckS+OgQ7bHxudwLNjMbcdKyElrpt5WHyWLrISvipu/edjnCvfZQNlh4Xi1MhaP5Z4fPeRKix+w6tJcSjXF046pNZBsboSREkuHFNN+2uM+5bEaLxf/EtqtoWV8BeSGVZgSQCiNAlP+wNkmpQxCL7F5qfqMTBBVHvJphyvlGCnksEyzYxrswnw8eh7ccwucvxNNQ5jD9uPwDeu3MGWrXWynjdhNit6O9+UaODXWQNxWOB1/lpewnEUG2yymMHXEHel4eZIu53fwYjtEQu4TH4oaF6CFb26a2L99Z1Ze2pG/M7UUQQTnkb9I68ILFS9YFaEmKmw426ykRVk1+mdg+RuQBCHEuyPAgTZRI73VoOTwUjgghiEfvk6tkGYAN7W2DOu1MeykdtY1SSqKJSsqO0OLsHDovewl1IGGCbvycxBxMtP54JIzSBITff6qcrizaAlYT0VNwnfvGf/Npi6X6bQ43FS7PpM+aLyXFGbgYyh0eCt5p/jVQpzVDnfaCl/SbNGS50066JOQ6vyn2Iws2jG1eBcAJ+Tcm/0M8/aH2EFhOQEufQP9Zto9id2hJg6SNAVY/hYVVs/QGxCfQjaSMxTpKWBZL7b+ykW61WmXfO+Qb/NC1qa3HcjPXEMLNWWqbvUt86A9S8zPZtVbD3fsavPA1l89Nk41j7KDmOxAeqnNAQ00jKjhbhJwpZxvOHO5jAlmXvT7LSR+2xRDsVrDgeelHvzA1H4YKaD00BwZ1Dd1fcYyUEI5xppWBYejxO1ron7cWQ+fWTyYFZlqGYoPpa1a8CYvzoGQdd8d5O8g+2EdpoOqM6PMpSdxnp53uO+8fKL0lGy6boHDMFslUmE1WmBE0I66T2p/tDarTcI1XSzfsIjgXOctTBQ5tO5/B4V3JAu6UlzIQn71d10nPHTtMWXHxCESqpYNV1XTaVU9Ze8zkgbp53lSd2+MAFlY/Zd9klmLNu09gQ8Y4NAVKRcttyC5T3sdZXuLDtYOfde8xmwFJlOrGuEfLu1MMSlE+N03uDLPQhYmInmn00buI5AJjFRjHyLNYPgqlDIcFl5LKbGrkbeMjiTwrk0/8Tr+SNuUO0YGad8AHqgk4T1YIyVLiEumxkx9QLGzPv7xmC3VCUJYA1VfUCxBviPX0hgT67dH3aNOyQuRfSqRunw/UHgtsjg2a0NeOoVH6wqYiulBzPvlrUZMBlckESynbigO50zjbyvEWCpSj79Chs+6ZHUp6fzqEbMsipd2bQB6O5AqDqzkXTEMAWVe1yEBiuEiDQCX7QcmTaBjbSBcnH0pMo9sOv03iKmR5vshAvm7IKMKWfk3RKxwRfOU4Zv1zLGnj1dR5H3K2cwRZjE2hVSigbvMOlLrSGptaHuC3Qx6kU9R09tKMK4lD2bCN7fBw1YVV1oSbjo6Hh5INEUBZx4Do5+c5qdqqk2LCMTLKUXhB8xEhPSZAi8weOtgUQ8mQ7ssh1ef+MzW3w+BiWxaInquMnqqppTRDDPWMG8dIRwUg4V9qTtNlRZuxsvwS4skx9hHEPc4xGOndxkpJPvDE1ktwfumRJPBzgTbDUsVHRa2Lvr76cxRXZSK/Rl7vNM62olgkwiCnnXruHyL16nHjGKxoKZIrhCwdv0ckV3g+qTKh42bHAe5opAUAkYI91oITetBMFWO1dKSmDrHoDTSMIxBr8LI0sOnt6qdSaxmlfxUBIbswdljNzXWTC0aQRWRjxJa3HrB/VA1Xyd5t6xB2ak8nWU/0fW9EHFkxTtVnZAFnRPTi4h3YmhAnrS1+z6H2OErgrNgQZEYKq3CeZ4Tux9n1/HREAcE59r1OLSK5yKNO4/6EjgjbcERos2uobUyBPOPyHRGkJBNQZa1I66aXKvbBPjQOwRf5RBwsdnxn5PtNXQV87n1095bisyx16JlofbLgzYqzOcHe/d7D+ykvLYO25i4ni8sh2aZH2g6r6HoG+umuCy6mNY615GVpt9zeaQG4ji4doIa9bdgQscYnEgQMTblWScOvOWiMEyfeAddSIOqcWGpQZh9Y6ARQ6sBKH33/CDixxFSJkvOxhJHBAqqeDJ/+PUfepaH0YUr5J3AR5WC1IaKIqPCmApbdcxpm1/dxQkY/2Z5CGmcEvzjrlMPIjmCJEJz6+v7kULafto4HTplZ3zYtogfdNQWxp6fhprq+tAsa+9w+rYzhoViCz2qntVkuOt4eQ1remSOCUvgPswCOjwirknyUaPDRgVY+InDUVVn2MCn7SS9SRhXn+6ij7wi93tFLWVo1wZbxnE73MXWPMdL/nDpGRRNY6vlwgec0MP6DW2VKlA6DGTfMYCua6unLW7ZDgGAnE87qkVgksSdcN87ilKqn6/IuQ7aVGAogPJSc2VZoFPNU6KNWewsNPYVsK5PDQzEycCTWRQgY8UsbEicaKOZTWkndFhFLjelgas6seMbXTHSwE+Z2vYOUDgqO2GPS+gzNoBQpB5FmWUiXfFUcNgAcI3jiQg+Mw37GD17zYM9VOUjIuDEeseWRkFEF2QlHxZmIqdjIYIqr6nmhz993YvTqXlfv5Ww8gzP7tEtl4utn+8Ai+yA78HpMdYfh0b2wzoTxiEIutpaFYRIiQJ02gnhpFTaVzGflYKSaLTgiVFn1kr7+/1b3HeBxlVfaZ3rRSDOaGZVRL7a6ZUsj2bFsYxubXgNrEkKySSDskk1gkxDAYIJNXUgoGyA9BMJPwmKyJCGhGiODK+7dli1LVrV6b9Pu/O+5M1ceyZI8su5kyfc8I83c8vXznX4OJ0uIROGjAzHlxbRDjphaenrPJjjCm0Gyi8TSmCY5vHIqbM8/baqluu4O+smilSJJnQiMKXfhfjF/aY6Npyd2fEwmg4FumVtB9b3dZGdgD7MkKCFp7m2jny66mrJbZ5OnxwfvsTBfvoDHmAhCHHHqTmqlU683k8YCbDdReCg8p7QgYyqwYwcnTkRaJA8MXaZDUjNV4oYQLOFrsdSd2ENPbttEtlh7gC8f1/d2rGeS2UKv7tlJdbomSr7HLqZKZhVZWMCO7SeyCBB2p/wA6rRmyI3+MkRqUCbnRAPhKmHDr8mEKe6v2qlxd5sYTHIqdduEgL4Jc8njOFVXuOZQhyO1f8QTsbRD3I4LmDRJ5JVdtLnWSBao0xjDR6Iwr9wFqf4iqNOAjGhzo5a1CBHkzcGSwBbgImRb2Xf6Y6pFGJkkRCXFoXxO4RE3g++jGCvds3cbpSUl0fKMfGocHkQq6AmX6pw6wr0gHWtirjUcMBsO7KErRdkA8oOJBOn5a2JhYf3AEDnzM+gy6zJEswCvakLNEaKMxGwrYnioYTp5ooZG3kZfszFrTJ6ML7gsYnVOnPgg3EdjOHGiSdSDi2zF+OfH/8YwGD8yZrYBQ791dAvawRpMsQws+4DEDhqNj8l4kRZJKmBiDCFeWNOJqfOd8J1NyPiLdtJmBgRwE76P/uEgU/XtHPD07HAV1zbX/WdwCBOe0ud0G9hcuZwV8SP7c440mO7dUOUmK7KtREKvzB1j77QOjnqaNUz14JXrsZBIyBjWITh+bcL5zePQo/6i5GH67LQB3/nAZRCTvzCL0AeILoXiQqs6TOv27SQrDFVYyj5RQddE99QkxuCQuu+uO0W3LFwqfo8RJfD8hLylE+yCI9pCrxzYTkMjI7R60SXUBqMdsQ/naQphEIFtOume8uspvgbBHiFbkPk8OtsDBlyoJpUJRK1RTdTwDABhsmwr0lt4h3O58axx4sSkS23id5xrgYvScxP8F7Er0iinPBBHtZoG+gMwdTLUaTxfk5V+VJxsNNKWqpO0f+A4pT6DLC/HwKubuPOTvYXruMcx4z0DUKd9GQkZ9yMh42EcYqBIRMOGSV5lMl87R6069cNGZHnpWgvYjQ9i9XPg+pwLN735JnqFtEW11v9CthW1PphtZZK2ZnSZG2J79rmwZ7eCV94EXhl5xiOGXUdZBESQ7RlSI1mBmmKgTovUIcYr6IWVRWlqF31w5EMsmkHU2U81aTwnfBDYoPL6ycHPSAeS+raSRdTU20lxEaKJRXWbyUK/3/4Jlc7KI0KMdxbWnbM5QjqeAPK+vruHvrZgAVWoFgAboeesTosQNhfDQ7mC2Va21SAqCOiOybKthPSThVjaEjVx4kQfsKvjoVhyH8KBdB6SmjGxwaEjQ4WGfr4LAjgjyPdJDuiQ5qgV7JAC83f3tg/JW+Ch2OUw2oEwL0Ajhz559jtTGGxxZ78jhjSIx3ZmHVI3gxI5r8srphyArfSSx9O9cST2ZFXNw1zrRGGnxqwlq9PeBFPf2Xny0hMdcTdAneaDXlkTKUBgvng4qE472GRAzHUQSrzTI1C4Wq4/HYEP0mxQp3GSCba4A4sQiSYlFmFJikB9Q7vot8dOwuzVSG1TYAQe9piDH+jx3WMHaMW8MqAyRDQde1e2WWLZQLLOQFvqquh0SzM9u2QlZANtE2L10blioiTKRd8ouIaij1pJiLQ6DeGhFNnItuKuo7ZHEHwRvCtjtPMuHjqMXOSkhnS3YX070iRZkMEUcgV2Xx2z+4PTyVXiEGApferqONqF5Ja7T9ZSikEvpqg636QzF5GAMGDU0U8bG3ZR0jdsAXXbZKaxwQXnf0krbNQcTMgYLmXE6jadU6OufY6xevcdPf4e50Rhp0aHygI4foAHcrzB8FRlDbCr1u+PFCAwkLPp6eJkl7h9tzTBZBG8ciR09DyJrLLrDPLKte069lTjrC0RAh2wcxiIGQs+K76ZXtv3EQwsYmGoMjHJznM+vrC6LRkS+Neh6+5DiqQfVayk7s5WSo6AYI7bbgYmUkPddu/mDZQal0CXZBeK6jaOxR5aeC5T4RzTitRNaxZfSkX9c2DBBSk7q5KCmzb0eTm+8yopcdD1Z3XRyXfqScOic6a/x3Zt8qbAPqngttr/92Hqrx+ilP+Euq0aJPVEWB1VexvhnXa5idy5Llq95QMywX+/CfMTbmmBnMVhtdEzWyupI66bEm9DdFhg7FGbf6kiHgL6wALDpHVW8iC6UOevcIiBAgk7giwgmMNOadQqT9v6fqrffeZJrh7LNobHGAV0yWYWaYfuqOpKnlfT5fLo2QVV6pTM/9kWH/WDVx6h7cCuUfgeMQEc+t4HIC8Bi2BGYMJP6nURZxE4d9pFyLZS07KFNjZ1UZpee4467XxT2sUiZgD772AqWzwrlywAxCYv7AzwYrh7/HxtSPd5nUXV2mAv7ag6Ql9esARRMfpgncetnS1a7KAmTneTaqIb0i4nzSFj5LOtQJ3mBxlc01JD/b9F8EWEhxrVK5/t2uTfGKtDSq1FcoTGF5E4MS2YOBHCrzHqNgY88Mq+NoESv2KlD09/BsntMEKKo73Ja5/wzgi/4dfTG4c3kfVKkyjglGzlR1/gfrEaMAcJGUvM1PAG1Gk2UCpI3TytBeaDbJ5K0/qXDl/PvpGVzT3NN6IC4O6z6jYR0HEhmG1lg7mqOWrthhMYqEFQRsoXm41VGBCWQZ3WNaimg11q5P2KHHZl329kr6UFHA8eLAJbxI1DVKNzP9MvDICcbWUWpLxxiM++du+nookpq7KmC5ycMSUJYZh3IoBjdXsLrWWSmrE6PMSmu/HCGVcrY6LYOHpm+0bSIab8bSUV1NzXDbv7UXwgkvNCbwv9bNF1lN6cJQqQWOAVqcKCbCWyWHTFt1LtH1pIa8HYx+uVw2kcZ6YSPL0HiRM79vZS6k1Qt/VD3RZCUjNV4oYALvH2WGq3ddJPt26iZJudGENPt3SDRUuLiaE/7dlD1ap6SoFbqwu28qPtMSxDSOeCOi31Ljv11cG45z3YBFxIBFnpwICUvu6nLdS8r4PDTnGQmFHvNnEFJVvZk3Wz1xxuT0gcckGdhkNiuoML53ne7C7wymkwqmBeeRO80xBzLmLGKgHTUwRfTHEhYoyCdjRrg+q7cHo7vWcY+PgQE7OtZA7Q7ppK6usXRtVpFwKcLSD31ZCK3w87+OT4BLps9hyqR/5y9hSLROHoswRDmnf2fkYri52ibICNQ7iYAfCn+4epYs4sWmleSnQEdG4kvdPAWingneYtHqbq4zXk/kCA2SvGDQw27cLAwIK5Ygi7Hu4itRGJEyH8QlCHgBUfKuRRqoB7mWx/49AngETYWIylgKfVbBsHEDHZ6IWdG0i3UE3GDB0JQ3zioBpMne+kQNHXckJGIzX8dwfNKIIsVwvyq//IIKvbcuuaG77PnZUodSVDvmgc4zqYd6Te+P2Pq93glX0RU6cFAIHJ2iGqbddSM/PK4J8jVViQyGxBYdIIbQOLYBRZhMi0xodKN1RACxIRQcx/mB47uJdiguo03LqggvWjWMaoI4NgcU7Ql9mCbQKS+oIqn+Alliyn4mB5/eAO6oGv+oMVK6gP1nMOSNmjeYd6u+nu8i9S3EmEhwK6jYwoEx1jwESmE1UyUYuuEckToAufDbJ2hsEemRvi0ryhk5IusYnfefcxpnUd8lDy2jg6oThNb+3bA4xspq4LwOZipfjDh2aqUU8HIczb1XOEkr8P2QAoCm6LWQRvH+YalEX77h7y1gQPnAsFBZ4vZk+K1Kra+5uoaU/HGsB2ouTdphzNtnLK8uTuFlGdhmMoMugioFcGrxznpWgErqisM3BCxojFg2d1WgdMTy+GPXvHgJqOdKtFoL/QuZQWcPL/LGdAtpWUTnrvyAbMPEwm0AcuM2mznXXdsKZ77uAuUum0dEfZRdQEt1Z7CEk9eZ+md4cBt4M3t9lOv962iYozsykNwSqYrG/s7aPbF1bQAv988lWjXvYIm8nApuiapE4bLuijE1trEWwfPWPhWRBQp3h16ls4iNm7rePX/SDdIQR7BEIwCMME8KlRuVivciU9CwxMJjt1ACOD/J26vinu8puNsC3Q2xLpfgj13Dkusl4VjbDQPqRgRvy+uwIJGc88BnUaUjRPS+4wSbswxVX6yOfp/nDYUlVV/Sg/xuo2JWdb6eqquvJ4u/06Dg8VrfWrIiH5lvrFcdTL04dof6Me5DrIjSAgSPfl+s+TzOq0rBgfJcd6qFJiESJkcccWZu3AQMvTYQnXv5NeOVkDaz8DNstMd2ZgRgZFklqDiCZ7aFlxKUi/QJIH5q+YEpTrw/PGsoE0vRH+8dVU3dRAP1y8AmmH2kFaqOhreVdT1GEzsq1A0h6huRQHMwjA5mwrQ6ep4/GBgDoNqiTG9DMqeJ9VUpoE2AG81k72uWbSVcDm/zCMY8Ar7+g+RFU19QgrZhDVadzchc4tv8vnoBX26tQ9TB9AuJfw5VgSmqFFsMDXYKlNjBHPsX1YFjHjsXF7kF9Aaq+uf/GMv2VXz7c6/B3zWZvGY6Cj9VFPfXxKT1a4aUItxOI6aC9CP6hgzO9w7519ThTAwbXwIvDKXujOt4FXZnUaA/vkdZ99f7rPsO93D0JCLQGLcLJVR22IDc9WeNOtJ5znUSnkDgIk+WrKsjfSK/s2wvwO4YYkOpEneYaFHWDSDCb6c81R6gSP/ijs4Ae624R4pZiqnbeJbB9sSx8MZnxqW6LvgS0bfA6r3TcnKUN40LmE8nsKyc1umDDZ9AMjhDM/031GwFwqYSbcl9FJJ98OZluRE/tgplQp8H77aIR6awbJcXMsmedE0Ug21JhbN8DVPAGYGIIzTCof0zP5cB3NCOHggFDvha2bqM3WQbabopFe2UwumE12vYJDrAwzzm60chQ0CKm9Qm1V+Vpf74UrazvnKSDVqfrTP9jVmPq1vVhZsxbqNJAq7Ilz9gMLJPG39J/vTfZ9/L3AbyXex9JxUkMxLtuW09HkhpuhhpnaMe1xvdJnfF1Sm+P/83NjPyr8HvIpaR5YhFSrl94+EY1MLyCrsXRn65+ontBrod/H1j+2PRwgGEePR0WXz3JRffu79NKxE6IpJOvC5Sq8YYZRnw9S+JqOVvqXYqdQ3d6urHbzCYYoe7CQ8kPZLNdHAAkIybvSOzygjENQtluWLlYsMy4U4g7EKQRIp1UQteMRUb/NOm7ZPiCNVLCAE+a4EUn2CLU8Al9zkNqQismC8cT14MkECa+OgyvrtiFKvNrqt14ZLbxx8mPl3tp6ioYZ6zBTYvycTMWHfQ4qngZc3XTZ5WUUYzNSzUstUNFgb+HQlJUFQteV8Upl/6dD3qh5UVmPvPxQjeLg8dozxxt1iX4wKTh5kYXl/G2ia+iXxJzxbEjfA3Nz9ldglvgJL8jo2GgYkUBKe6pZiRDOzM/y/bPvj6137D3+xc8Gnmbr9NA+SHcDPeFn3KAaMhMFGgQ53dYD+3mo76T2pDrGvhX4xX9D6z7bu7P3Q7+JzwLbKmGeajN30jV//RkOGY3IkvDwxs9F6LvT/+6nLKjWajpahKdXXKf8QnZOTXNnR49OrQG7hT5MUCEfNdL10GOHr0m/A/fPHSnfVwGYR9xub9GsrDhNmzFtoMElKHU4VNAev8GFx3ju22evTfXchO9BO6JK8NHux46R+zN4wyXgKaZX5CzotDIGCGG3W8h6JFGZ9c1k2vjZTsGo0yl5LiNRGLiGXC4qzs0m91YfHb2jnqIWQBIfiXjwGAD81n0Gq1FV9GLqfjUA7qXqqt41v/ygw5drUyiRXEP2woeHG2F9zEYlfesGM/3tvV4EexDIBMmjTCzsmD6D6qMzfQJdU26geQU6uu+3PVQUB5dFFjOOeVKeH2zxeKxDoDU3WWh1YSk9tH0rJcZYJnVeuZBWud98qNR4sEJxiap8R8qpRfOccyEsGryQ+sJ9BwCNJkTji4QPjm07/MZ3D9p16Vg2H+dGkb+otKCOtrvp4p9kUwZivp14t4n06VrRsETuxYNFmWAkgzKmMKoWXPLuE2eaVv362H5fSVSMygXtg9x7BfOIhCQC6fdtpofn30wxBQZytcN5BWOWFSPgnBLNePd5/VnftpI5Oepl9ezMzEedRVW3DG/UZ1T3e0Dwyh/dlQ9IE3JubT3ko4UgyS6piKZ/e64P1l4gnWCHDEpb9qKDNdNT77no1Rw93Yxg+8+8P0xFDqVozCL3CvKOHwb5/nsYPNx7awkV2Q/TYZgz6nBNLtaLcUwazF/rulr8z19xE5XnFT4gAvmyZWqIVXGERaYwkBPawP/W000ND6+42/vCu/ce9NlnGZReWJAxGyNrQXXWWDXtuKeBMv5iJ+uXTNT3xjBMWDHLco2SAQHGKiN73b6iZzKUjnLL01kp6b9YWjDnkp/u/tSyzQchAajbwLh45qcaI9/nMtUzgScYmtVYQ29zG+3KO0Jf+M5cOv6dejKU6RCbnjslPTfD/0A8yNnuMy+MVlsrjPsyU9Of54SJrrlzotY8dL2FalsEfyz4L4TJxjkn3we2CUABfirOUNKPAHDmGBXdukJLB7thUwy+Xc62pLo4mmusRUF//3SIFpUgggwCB/I8GtAX6Rm5/nNbWVD97GgYoWNHDfQf85aA9+qF+guNyVTY5rzONexdnpWvLkvL/DjeZltP66Cur6yMAA02ttNSGxnJqS9mV5gPJJXEahDEwauIxdYFOybrB6ydIp5hWqD9mxvIcZ0N33CgyMnHYr0EGAyYS6I10eXqI1mp6S/xgebMz1/72CXXEvW0C2kIAEKwHVABMFXif/4+0Yfvn++ZwHsM5Ey+R9sS6MGtH9Fw1hCi0ZjJA5dUGPDLU3BeKA1I4VTvVaTcbqckp/0+rhhD9isSral/LCuiTVfPi1HXD/i9TPryOSX3BwgOK+inzbuG6JKFSO0HgQgX5gXlbovDa2fj1F6/x0Nn2rx071VGOoRYZgbIdeRuj8cwCGutYiTfW/NOP9k0ObQqO5OahocplvkWGYoVbDEND6i+XuSkhflF93CV/rXTz6p5IV1hIGDPRn43Z57j3oq7MqgbWS6U7GfNxwxPqFwfbAQB2C0mX0v7nmlBpNURSvwhXEvFSK5ojzfKTAreZ19zd62X0u+MJ4cz/l5GdqhSmZ6U+nzF7IID6SmZ6nr3sJft+nlYPEQ5PkyQuEHe6iG8pH43vXtqe0DdBpUYG9DMeGyoXzTj3evzJX09Xmkvi/5Tgj5hA4dsV1ZWbhLRjrPIdN+/LFVRW6dfDQwVEXEE+0KUIr7xi5UucoFkX3OlgfbB88kobiH0UsbC4MWplXNhpfab9wcpM1VLlxWoqBU6VPHAkbEtqSoYF2OX+mjrDqLrChbjO2yXZaDHAOTUONTvvaNkoWL+7LyfK7TavQx42KC8//4hhXWxvGFspoQPs0vtf5r71RTV0BGvZ9oBEMPpLWNcOHYYYSe648+nyVqB2PdmSOLZ2QP3ZlSAORFK2ev4arzK6oz6qyMq/t31q9arWBDN9TrzClbfv3glnFnalBzCi4WOchcOO5VsiaVf7NhCzdHwXbgLWV44mCSIiBkV7mrAMEVtu8rozp6TcD/Xt2rVKlLyAvKm0WodO51zdL/9LkLM7unweznyityFgY/15naQ1G9VDlJpoZ6y7ZBEAhtGovC4WaW3v0mgg8dd9KWVUQjOCP9djE3+5YOBDg6WsngVPb9xgHz9yXT3nHmwyEMSPj7BZ1DQVwFRAjWXzSrozM+a9RBXtWzZMkY2/9CCDSNOW3Fxxuri6+NcHhI0Cjb9i8BkspWYsRAaBqQgakRARnYKcR8HMAAbz6Q95GeHktKvtl9j9GQXp98rTuB6YEI4gKxbt05pNpvfL8vJ+/P1JRXK+oE+j1lG9it0sfrYQkZjEm0uolcaRNfbGWljsQYipXLA4818MJmSymxPI+tqNQd5ZYQg7kBp0xTlmh+6uMzXh/g7iLuikNNEYXSMbBmXAZLv7QNeajzjoTuvjKJjTT5kLpnR+o3WP/4Le5KVIK73uneGyIS0urcjlteebtgEzwz2xjcj/maEzvkFDTCh/uvHLlqQhvjsiBjSAoN7bo4PuumWRGy07oFu4dHyi2h+TsE6LFpncPH+4YCOtoVA2/pTuWUpP6lYM4v6qtxe5UR+3dMd6PjnMWFeOARZ9Dra9rPTpJutQ9ojPflwaLNDyIUUJo89+73ezAeSyTHf/qxeoT/B47lJcZNIGRUUFIhL5CyY88CNxeUC/As0JsaQEcDqHHYqzWSiDYcguHUhKMmT8G6Ddd6od9t0B8jn37Dfp4/RayxL9I3ZmZlPcBVOp1Mcm7jdzy5g3JmSItMTj19npwONPh/26EwOzwm7yjPJJHVekpJ+9f4QpSdr6Mpi+AkjWkgkSGpuT0Q4+PLxZ0O0YgFkAwA8vh6JwurJwmglvbV/mFobLfRUyUI4w3cjYAQcMqbZIIgRZFDx+FJtCepFWTkHkx2OF7kKLB5m8P+mSG1np6c/kb/MVh8FCwWEXWJ7KPkLKD1NFhIoHhqhqiMtlPJVO3laA04hIvM8nRax4NArC1qtVmNdqm+elZHxGL8ujYe/c8jkIEt0vLyg8NnvL72cmno6PfEg4SNRxNj6iBF/17YPSDHXT9HzDeSDgBr83vQKY3MgT1eV15/5WBIlzI1dC5geDEUIo8sjDTgjJe25+UWe6txkg6YfCvfRB6bX9JRP84Znvvxou0D7jo7Qqouj6Ew7hBQROFi4I6w/L4ce/Zcb3TQCPu/Bywy0HwnsItEeY3Um4WfhIHvlgyHKthRTucNGDSB0Dbg5nQMG5q1iYMjVpRXknJ0n8lvgGZk3n+6ZMeV6TOcmtx3cQMM5JckPLfnpLOpucAsqTk8sd6+w+Xy9frKkwyd/TSN5IeW3/2v0GNfSsPrOgACS33XMK2Q+xYBgY8poIDiOMb0GdStiwNzM7MeX589pI71BCwFaRKgnPq3FLC9NHbS5ZT8lfwsHWd1YH/mwxodtghTRXusys9pSrt+ZkZj2O35Pgmn+PgrHvICVleImchflGdd87xoLVdX5/SJJPWYq+LWZFw4EUQJB2SNQtxnhtvfvy7W0G0Y0kSKpGfhiIQ9g2UBJgZ5yAfiDrMOf+VDOqYFdY2NgBLGn1U1HjujoW6xuG+wmBtxwGVozDoVG15D36tkFqvLMnL+Bd3wXUhUxmMA5Df6DL7DrI5pUOOyO32cvsG7PWGJTu49B3cYkiNyFgRRzyfO2/5MGSrzSGphDRrLh7ks862v2e2OXxKgt8w270xNTf8PdDAUEqdsMB8HDtKcsr+hHT624inq6zvgiFcKLs7zYkOXlMajb+lP7Ke5LZjHsFCJFh1dwBCmjlH53q1eZcquNkufFreYXg5TJ6AyN2efLlysADn6FIz59fWm+f+P1Cy3q6gHycAYVuQtjNlEIgLo37gBJ/QWQ1LyocjcUrI+FgLOAdd5GBBGWDXz7ciNVQeJvYNnA6HTI0ziPgQ+W4gQlrf1gAAEbZtNXcmcj7dMgWcMMx2LkHEPuEfWq3LlCeWFRAJuvh9Toc1IqEceAu5LvTLmv/NYU6iWPSmmEClBu3IfJFMDWmXK1dOj5Nur2DFPSaiuNcC5yFsydr2BtWa/safEoU26zkaPUKgLCVJQRgJ2RrSIpIeHXC3MKds1Jm61p8ox4OStNJIqKeVZog96u2kpxXzQHYCDMgyyYZMKbdptDaSuJfiPOEFfJ2hEWsof29RwQxqYXr5UUGaFuQwifdkGtg2O33MDAnWDBVblNSb+Gum1gUKB1VxgjSlKzdD8Huu5fQjaQmqSha+ZqqBrRQNmENRIFbjHYLj76dLufrs4HVkckBYhAz9tUAjD/mcFe752li1gA99/YeEeCJzRm7PNRlgMYeEPFaKybZ5clvOa8LU0xcDQomJO7i5gyjuSKYNn02f+eJsuCGISUCqrbztOWKIDb4/WmfDNRaSs1rU80JG5kdVoQmCd9G/Mt7orygjn33FmxnKi3SwkDKJlRQqD5Nvj6p1is9Lud26je2ExJP4S6bX8YGgbuDbuFwpwo9iqDK6ko7gGuUdKOBGoP/D0H0DEBokBCq03aUzrH8IvvXxOn2N0ueOGBKXvhPc+YzxGvpNc/GoSxv45KwduyWWwkBHOMbKJBEh1rDcgGboRsoKczIIGPxEHmQoNO+HD//JNBcvU46D6EbW7t65tU3SYdAX3sp6k3aFZm57XmZWU9yhMvaUZkXwQZKiwqyllTeKUdQZJgv8Y2JhEAB86Gqs9XUd1f+qi+GcE2EQnGzckRppL4i4DAhKNCY73K6IJeOQAI61edd9SMEddB7WwwGD4pzy34482lFcqmwV5fRFJZozf9TAppzfQSsrxEXaSDUc35s7yI6rT9Hm/2I6mUVGJ90qKw1AQplXPoqnMAnWdA2lT5s/Rrlzt9XUB58P1SnPPyeWcrjAeYpE7BYm1AUP3aRjfdelkUVbdERtfNgMQS/3mweX8MsgEd1C3/sUJHOMhEwVwY3Z3WI9weUy0OO9GfNoxQeQrUbcixLanbxlfG+zIV0vnhvi7hyQVLyZmdy9LTnlDp6fh3/i9/s5Q62Lf6HKfjySXrIJg76fKp4BUmO7CjSh8ipprjoW575jTyjmkp5lJIqVndNglFJoZS3u/xZa9NBcke+xO9wnLKH5BDhUUZFbRDQowyL6/g/uuLy4fI50Og68jAASfRSEUq5k+RTvvAUBWlPIUsL8fBnrD14UQHJ1+GOs2QYdBYFutqszIy/iu4FyYc24SAjs0V1Jc62kuKo9b9GPbGB5AUjKXUchfG6mw+WoAMkk9D150Yp6Yb4X98nEnqCXsnQw/QJutiP9o+RMvmQzbAk4bJnGg+Z9oaLH7hm66kd44OU+PpGHq2ZDEk6cjIyRL1kMJdYvlLg8fjzXWkqRemz96TkpT0K35kIqERX/88FKlv2emZP85ZYq2J1UdpvL2wBonE2mELq+CzMFDjpqP7myn5K4jkirBMk+meRUCw6DXmxdoG6JUDgLBMtGYNa+pCD7KywqKn7ltyKXX0tPsScRhHonAgULLE0w+2bxBTNHOq5gmzvGBPiTHuoE7LuC+RkubZ2MHJNZXcYdLlkBYwNTH9hQXFwsEFGUYNSGr2n5e9MICxDfqZHj/tPjRM114URYNQqzCgR4Skxnw6Y5X00mY39SPf1cOwg2fruUiZ4rJsoCAFFnN/H6D0mCJanpxITW43hSZH4DngPOPU3624m9VpufkTSk9ln/wZVogNhv0lamtc2SWJDy55Npu6zoz4VWY+PWdY+fjXUSXbwZthPLNzXRO5TD6Kvx2RXMcnR2BAACYcOe7xZz2ezOq0H6GfSEgzfdWkBAdZKWlPLc0vOk1mq6bF6/UxqMsNChxtl1MyI9k8fdK8R0zV7OVoPuPZE2wTX73gtV9tUdvKoyqTrcn/s47WsYPaGAFc6PRNCujSAvLD8+YY77/tcpDUZyAQAfMfgfUT3UdLQFI/+eEIaeHSeidI6j1wg4wIFYExMUmdguSHb348SHMgGyiCbGAAACn34kmTrYfQoabXQ/v2q+mWYmD1kX6ywIadC7cZgxO0bnjAc2P+PJUzI/stk8n00aoJpKfiC5+zP8ENpsCGez3Tad08a0W82nUEAcOwZ2UvPFnYgOxDsGdDHcVfAuU6XwrdyYBCkPRe+8pYtdmp25Een/J7PIJuTt83gOEgyJ64nFC3Pbv0MtjBt/iTQMSHqyoVOxjmH47/b7faxRTNPYm9lPBVi5i6eVR1ibEj8Iff2+lTJUIV53DaRYSwtBKS8ynKlDd5AVdBQmmOSn23pED5t5srLKrDkBRFgqQW149PEOhMP0R4nyVOkNQ4uVgFx5flLkxSJ0B///4xH9U2eOiOK6KoCql4jBEyxWXZQKlDRY+9109RQhbdmpdHzVC3cSRXHl80NhR5XJobcot9pfmFotBofdC2XO6xR6I+YEtxL+WVOu6b/40UeJ3BCdIUAXUbOi+q2wo0dORXHdSJlNJJDyHlESK5iiQ8WHZW83lavMrkr1vZTVO0Zw/274K2koTVE+z218qz87bOn1Wobhge8hqDB7Xc8ynmqUQ4xz8f20L261jdBjsCPjQxNs7v7t7n9aZ+x6FIcFp+Z1fYd7JGZrw6bXyfpgR0fnh9UEJZVmxdfdVC+GP1w70C7V7QjI1vfdzvgJRaQS9vdVMPYl4/CpL6QENk7OD5YGHgKwQV8cJ72CwJarqhLKBuE3OGj+vbTH9ye2I0Hb1AG7d46ZLZFUBLXsTOIxjSKKlpoNd7N+zZnbNzn8YBW1Xpr2QLONz95yjoq6itidHYts8qi395/rczFANHoG4LR9c93SFi1wqIYR4NV5Btf6wlc6mJdDkaJE6E3B+Ht2ePx5v679Arl0a/Fq+J31y5TpxL0HAXVjA2PwMTv+0sKLzv3yAopd4OlahuiwAkdGGjcIrm/4dUzbXaRkpZDTt42H+wlB0BORFtUaWxXWroy81P+hH3SRKe8/fJynkBPbCAzNvYjpYXG5+79+o42osUFMxTy10YGJikTgVJ/cbGQSoEL1aeAacO+OvKHchE6rsWZN5pyAZ2HhymLy6Lgp811G24FpGDDGMrs6rot58O0kBnPD04txyZXHoJhhgQEJg0K7LzGnMzsh7jvi2jgCmm1M9/hv/ShivIm72m8Iq4PuBWBM+T3YRGnApWt+mQOLHpg36qPd1Bqd+Fd9spNqLh9pQa6xWGwYyCZJEyWrZ22YwPTMaYbDdgNBq3OnPyX/36Fy5W1Pd1e20RsoMXUzQbLPTLXR+RYbGWDHaE00LyEeRN92U9AblDqe0JhSKqOVyNzHkBnWd12bLAYuVkZz56Uam3xWDSauC8gXCP8vO0bD4aj1N540kfVZ+Gug2ygVNBQVmkgM8JU9wnOe8VUPn3VkI2gPhvkXDT5cOK7e4diQiIAVnEPIeT8uzR1NDVITwzH9g8O3cNDtYJbbDF3f05/4O+B7U1ijO5zqRHL3p8FvWwuo1VRFykDSPHf+bDIZiLTdDTtkfqSJmqptgbo2hoh8uXDQFccpn9caPC2BAuIAQ6OPVfyRBlXl7+g1cVlQ6BrtBAnCQeIgxIcgyL6+C6OJU1Z3n57EQ17e4/CjfdOBrZ4/bFlJs05oW6KggHn+beSmwFf5+qhAXoIQvYO68oau3j11rpUJ3gQ3BHge8hIk3ww98n+oTe5+/jnwm9rxDcgkKYm6wUnn1/WIi3a4SbK7TCwT6/oFdP9B5fG/t+4HfosxO1efY+u+RqTArh/S1DwmKm/dgZAAAC4klEQVRnlGBAIFAcOIJ6tJ/j65faPFtHYEwTtXP2mhL9xGElpJlUwoaTI0JdTbRwx5wFHrJY1BVZuVsSEhJexWIpgrbkU63b5/aetPEQreWZ2YttR23ZJo23y+9R6rFXkJZc1g/mUx2nFEa6vMLRvc1C0vV2jxbedJbFxhMZyWk/5kmS+iPHhPFerwxI7hvK8gsef3D5VdTS0eJJV2sEPe5FKZSyfMD7CyZ8enyCYLMlCvdt+VDwFfgEk9rgT/6mnZKdCfejL77paBH4AAmroFLUHfCa2vzZid0Pv+JzflTbQ4nQSblFo3WuRqouFPfytdDfkz0Xeh3CKbAGdbBgu+tSHS2ca6Cbn0PwAasS8bZR22h10pfztSHdD/3P7QXeZ+YtAULAE0d99IvVMdQPc9x7XxmECy1ycrPUTiz8rlTCvcbPS88G3uVaMI9Q5gbyw796h5Ji7N20PLdsodlm3hGOYCVQ0+f3L2NRPqy6Btuv3vQ/J/725m37kT8OYY1nTECfO2aeXZbA9yIu2aqXCik71UHGfM2N2akZb0n9OPetC78iwQH+az7ase3YFetfzvb29UBwjA17dmNeeAPj3+R6O9rp1kWL6XsFtyBDzvCHJYuLLmN1Gj5hz2jo7h3fxDm/pU3YcKZ22cla1+t9fb5hSOYU0gIyeTBZy6H3QsmI0Oel69I1YEAEvYeDfpKOWjs8CJUbMMTn+6H1SR0NvSbVJd0brTN4YXwd3BanHNMB4OOsGmo44xKljtKApPel+vh/aBuh9ye7Hvoux+VzeQSf3ao1Z6Ua/5CSlPI9bB62weZz55++rEO0FnyEA4eOv9pa33s5zrY+iJJU4+dJ+s1zJn2XBi9dk/5L18f/Z58Cr8/nM8Xoo1Nm2zdmJKV9RQLI8c/K8VuCg9r6+uuqG+t+MzA01KdUshWNNAKpx5P9n6gXoc/yfa4rcI1zc3g9XkVOZoYyzh57bUJswoHp7pX/D/uvvx0CG0OhAAAAAElFTkSuQmCC)\n",
        "\n",
        "Let me start of with this questions:  \n",
        "*Do you...*\n",
        "- want to stop manually putting all the tensors and models on their devices?\n",
        "- need easier paralellization?\n",
        "- enjoy functional programming?\n",
        "- want to stick closer to the math?\n",
        "\n",
        "Then wait no more, we just got into `JAX`.\n",
        "In this annotated notebook we quickly go over the basics of the newest contender in the computing world, and later be able to implement our own autoregressive language model (Transformer).\n",
        "\n",
        "***\n"
      ],
      "metadata": {
        "id": "7Tj6FHt2_Usq"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "JAX is Autograd and XLA (is a compiler for highly optimized operations), at least that's what Google is telling us. It is used to enable high-performance machine learning research, or let's put it like this, any kind of numerical/scientific computing.  \n",
        "JAX is able to automatically differentiate native Python and NumPy functions, whereas JAX has their own implementation included of NumPy, such that as long as you are familiar with NumPy, you will be as well with JAX.  \n",
        "However LAX is now also targeting Google's Tensor Processing Units (TPU) which are custom hardware built for machine learning research, and thus highly optimized for linear algebra."
      ],
      "metadata": {
        "id": "vmVscHJGBVyy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Some Caveats\n",
        "\n",
        "- random generation works differently in JAX, we need to explicitly generate a `PRNGKey` which is equivalent to a numpy `seed`, however the state in PRNG is deterministic. If we need multiple keys, we need to split them. As everything with JAX, it will return the key and a new subkey (state and update), because you never use the same key twice.\n",
        "- functional programming, no side effects allowed, this requires some change in how we think about our program's state. I strongly recommend reading [Thinking in Jax](https://jax.readthedocs.io/en/latest/notebooks/thinking_in_jax.html)."
      ],
      "metadata": {
        "id": "CvUdS4rBb4v0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "But JAX offers more under the hood.  \n",
        "Simple composable primitives for machine learning, just in time compilation and paralellization:\n",
        "- `jax.numpy` is the NumPy interface that jax provides\n",
        "- `jax.jit` compiles the supplied function just-in-time for an impressive speedup the function is used often\n",
        "- `jax.grad` takes the first gradient of a function w.r.t. the first argument\n",
        "- `jax.vmap` enables functions to become automatically vectorized versions of itself\n",
        "- `jax.pmap` for simple paralellization, and differentiation across multiple devices\n",
        "\n",
        "We will go through those primitives step by step to gain a better understanding of the underlying ideas which accompany them. Let's start with a short intro to functional programming and side effects."
      ],
      "metadata": {
        "id": "sV3p2n8SC8bI"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Side Effects\n",
        "\n",
        "Side effects are everything that changes global program state. This can be as mere as changing a variable outside of a function, or mutating a given argument. JAX requires us to write pure functions, which do not change, or mutate global state. It takes some time to get used to, especially if you are used to other frameworks which like to keep a stateful context about the parameters as for example in `PyTorch`."
      ],
      "metadata": {
        "id": "dSgBt0CZeC3F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# impure func #1\n",
        "state_var = 0\n",
        "\n",
        "def impure_one(): \n",
        "    global state_var\n",
        "    print(state_var) \n",
        "    state_var += 2\n",
        "    print(state_var)\n",
        "\n",
        "def impure_two(x):\n",
        "    print(x)\n",
        "    x + 1\n",
        "    print(x)\n",
        "\n",
        "impure_one()\n",
        "print(state_var)\n",
        "impure_two(state_var)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8J9OyudZd_Oa",
        "outputId": "d1e866de-0218-4629-ef06-dca8346b89ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\n",
            "2\n",
            "2\n",
            "2\n",
            "2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# this is fine, because we don't change the variable outside of the function but create a copy of the value\n",
        "def pure_fn(x):\n",
        "    new_var = x\n",
        "    return x + 1"
      ],
      "metadata": {
        "id": "0CwmdLgZfbOP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Overall I recommend "
      ],
      "metadata": {
        "id": "CVDvTzSpioPZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## `jax.numpy`\n",
        "\n",
        "The NumPy interface to JAX is similar to what we are used to when working with NumPy. Perhaps the biggest difference is, that JAX arrays are immutable by default, such that the following would result in an error:"
      ],
      "metadata": {
        "id": "AzbeLxQOMw0R"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "a = jnp.arange(1, 10)\n",
        "print(a)\n",
        "# this will cause an error because we are unable to change a vector inplace\n",
        "try:\n",
        "    a[-1] = jnp.arange(1, 3)\n",
        "except TypeError as e:\n",
        "    print('Mutation is bad')\n",
        "    print(e)\n",
        "\n",
        "# this on the other hand is fine if we really need to manipulate arrays in place:\n",
        "a = a.at[-1].set(0)\n",
        "print(a)"
      ],
      "metadata": {
        "id": "ioUlN20uMarb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bbde6178-539b-4ac1-ec66-fab96ee298a2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 2 3 4 5 6 7 8 9]\n",
            "Mutation is bad\n",
            "'<class 'jax._src.device_array._DeviceArray'>' object does not support item assignment. JAX arrays are immutable. Instead of ``x[idx] = y``, use ``x = x.at[idx].set(y)`` or another .at[] method: https://jax.readthedocs.io/en/latest/_autosummary/jax.numpy.ndarray.at.html\n",
            "[1 2 3 4 5 6 7 8 0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can do matrix operations just like we used to do with NumPy"
      ],
      "metadata": {
        "id": "OvQ9GctsWxPd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "key = jax.random.PRNGKey(42)\n",
        "a = jax.random.randint(key, (3, 3), -1, 1)\n",
        "b = jax.random.randint(key, (3, 5), -1, 1)\n",
        "c = jnp.ones((3, 5))\n",
        "\n",
        "jax.nn.softmax(a @ b + c, axis=-1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U-iJpAMQWwfJ",
        "outputId": "d9f35545-c695-44ac-822f-1cf76e6a7dd2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DeviceArray([[0.11853155, 0.11853155, 0.11853155, 0.32220268, 0.32220268],\n",
              "             [0.2       , 0.2       , 0.2       , 0.2       , 0.2       ],\n",
              "             [0.16430707, 0.06044517, 0.16430707, 0.44663367, 0.16430707]],            dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "So we have just done some plain old feed forward as we do all the time.  \n",
        "Other than the mentioned immutability, JAX arrays can be used almost the same as NumPy arrays with other packages."
      ],
      "metadata": {
        "id": "4Hch7ifsN5RB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## `jax.jit`\n",
        "To gain a speedup we can compile functions so subsequent calls are faster.\n",
        "Let's look at a quick benchmark.  \n",
        "For further reading check out [JAX vs NumPy](https://jax.readthedocs.io/en/latest/faq.html#faq-jax-vs-numpy)."
      ],
      "metadata": {
        "id": "6A0rcItGlmtW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np"
      ],
      "metadata": {
        "id": "l31aRrqtmKC0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.random.normal(size=(2000, 3000)).astype(np.float32)\n",
        "%timeit np.dot(x, x.T)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3GV2gM-4nUAe",
        "outputId": "b1c3565e-3c29-4c77-8fa5-1b97ec6d3b14"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1 loop, best of 5: 412 ms per loop\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's compare an unjitted JAX and jitted JAX example."
      ],
      "metadata": {
        "id": "ZGy_dAVloBJJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = random.uniform(key, (2000, 3000), minval=-1, maxval=1)"
      ],
      "metadata": {
        "id": "6SLE8VCSpgkd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# unjitted\n",
        "%timeit jnp.dot(x, x.T).block_until_ready()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IluQu_czoAXr",
        "outputId": "ebd5093c-5e32-4fac-ba07-6a92d7d05c35"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The slowest run took 161.97 times longer than the fastest. This could mean that an intermediate result is being cached.\n",
            "1 loop, best of 5: 4.91 ms per loop\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# jitted first exec\n",
        "jitdot = jax.jit(jnp.dot)\n",
        "%timeit jitdot(x, x.T).block_until_ready()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4bnU8d1boV8o",
        "outputId": "5c2315ff-ae71-47f2-ddd8-ea361d392632"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The slowest run took 138.88 times longer than the fastest. This could mean that an intermediate result is being cached.\n",
            "1 loop, best of 5: 5.2 ms per loop\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# jitted second exec\n",
        "%timeit jitdot(x, x.T).block_until_ready()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YiE2uQq6pRlZ",
        "outputId": "88e2144e-75fe-4a27-b190-923223dc83da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "100 loops, best of 5: 5.38 ms per loop\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## `jax.grad`\n",
        "\n",
        "Gradients are key concepts in the world of neural networks. We use them in backpropagation to actually be able to adjust the parameters by their error. Taking the gradient in JAX is as simple as calling just another function and is a core primitive in the JAX ecosystem.  \n",
        "`jax.grad` wraps another function and computes the gradients with respect to the first function parameter. Let's take this small neural net below as an example."
      ],
      "metadata": {
        "id": "o09WsVMiRwg7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def relu(x):\n",
        "  return jnp.maximum(0, x)\n",
        "\n",
        "def predict(params, img):\n",
        "    from jax.scipy.special import logsumexp\n",
        "    # per-example predictions\n",
        "    activations = img\n",
        "    for w, b in params[:-1]:\n",
        "        outputs = jnp.dot(w, activations) + b\n",
        "        activations = relu(outputs)\n",
        "  \n",
        "    final_w, final_b = params[-1]\n",
        "    logits = jnp.dot(final_w, activations) + final_b\n",
        "    return logits - logsumexp(logits)\n",
        "\n",
        "# random init for params\n",
        "def random_layer_params(m, n, key, scale=1e-1):\n",
        "    w_key, b_key = random.split(key)\n",
        "    return scale * random.normal(w_key, (n, m)), scale * random.normal(b_key, (n,))\n",
        "\n",
        "# Initialize all layers for a fully-connected neural network with sizes \"sizes\"\n",
        "def init_network_params(sizes, key):\n",
        "    keys = random.split(key, len(sizes))\n",
        "    return [random_layer_params(m, n, k) for m, n, k in zip(sizes[:-1], sizes[1:], keys)]\n",
        "\n",
        "# remember we will take the gradient w.r.t. the first parameter\n",
        "def loss(params, img, tgt):\n",
        "    preds = predict(params, img)\n",
        "    return -jnp.mean(preds * tgt)"
      ],
      "metadata": {
        "id": "wUYvXRUsD5LF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "layer_sizes = [144, 288, 288, 10]\n",
        "target = jnp.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=jnp.float32)\n",
        "params = init_network_params(layer_sizes, random.PRNGKey(0))\n",
        "image = random.normal(random.PRNGKey(1), (12 * 12,))\n",
        "preds = predict(params, image)\n",
        "print(preds.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fKpN0h5IeA0v",
        "outputId": "f0d35499-06db-415c-8344-0caaa96aed8e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "jax.grad(loss)(params, image, target)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CgqCn9DjeUOZ",
        "outputId": "2fed3e48-ea16-43a4-82a0-4dd5d5832b03"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(DeviceArray([[ 2.6296640e-03,  8.4307256e-05,  1.0911950e-03, ...,\n",
              "                -2.3939245e-04,  1.4109161e-03, -6.5495161e-04],\n",
              "               [-1.3404343e-02, -4.2974442e-04, -5.5622132e-03, ...,\n",
              "                 1.2202694e-03, -7.1919467e-03,  3.3385239e-03],\n",
              "               [-0.0000000e+00, -0.0000000e+00, -0.0000000e+00, ...,\n",
              "                 0.0000000e+00, -0.0000000e+00,  0.0000000e+00],\n",
              "               ...,\n",
              "               [-0.0000000e+00, -0.0000000e+00, -0.0000000e+00, ...,\n",
              "                 0.0000000e+00, -0.0000000e+00,  0.0000000e+00],\n",
              "               [-0.0000000e+00, -0.0000000e+00, -0.0000000e+00, ...,\n",
              "                 0.0000000e+00, -0.0000000e+00,  0.0000000e+00],\n",
              "               [ 4.0973131e-02,  1.3136022e-03,  1.7002048e-02, ...,\n",
              "                -3.7300042e-03,  2.1983664e-02, -1.0204884e-02]],            dtype=float32),\n",
              "  DeviceArray([-1.43490918e-03,  7.31424801e-03,  0.00000000e+00,\n",
              "                0.00000000e+00,  9.74586525e-04, -1.89102720e-05,\n",
              "               -0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
              "                1.06202590e-03,  0.00000000e+00, -0.00000000e+00,\n",
              "               -2.35239528e-02,  6.75391871e-03,  0.00000000e+00,\n",
              "               -1.69281326e-02, -0.00000000e+00, -1.57179050e-02,\n",
              "                1.34678651e-02, -4.97317640e-03, -2.27013379e-02,\n",
              "                5.90373250e-03, -5.41177066e-03,  1.07336398e-02,\n",
              "               -2.00371221e-02,  1.02741178e-02,  0.00000000e+00,\n",
              "               -6.34448789e-03, -0.00000000e+00, -0.00000000e+00,\n",
              "               -1.45123340e-03,  9.96302348e-03,  0.00000000e+00,\n",
              "                2.44663097e-04,  0.00000000e+00,  6.35170471e-03,\n",
              "               -0.00000000e+00,  1.49396211e-02, -0.00000000e+00,\n",
              "               -6.12843456e-03, -0.00000000e+00, -2.06695274e-02,\n",
              "               -9.06322850e-04, -1.09448060e-02,  0.00000000e+00,\n",
              "               -0.00000000e+00,  0.00000000e+00, -5.23743220e-04,\n",
              "                3.28686484e-03,  0.00000000e+00, -0.00000000e+00,\n",
              "                4.52152593e-03, -0.00000000e+00,  1.56155163e-02,\n",
              "                2.77762003e-02, -8.64079036e-03, -0.00000000e+00,\n",
              "                1.74856596e-02,  0.00000000e+00,  0.00000000e+00,\n",
              "                4.20271792e-03, -0.00000000e+00,  0.00000000e+00,\n",
              "               -0.00000000e+00, -0.00000000e+00, -4.67677694e-03,\n",
              "               -1.12489816e-02, -0.00000000e+00, -0.00000000e+00,\n",
              "                0.00000000e+00, -1.80422217e-02,  8.35313648e-03,\n",
              "                0.00000000e+00,  0.00000000e+00, -1.39911892e-02,\n",
              "               -0.00000000e+00, -0.00000000e+00, -1.75164100e-02,\n",
              "                3.92625015e-03,  0.00000000e+00, -2.14883201e-02,\n",
              "               -5.22751641e-03,  7.44116865e-03,  0.00000000e+00,\n",
              "                1.47976503e-02, -6.21382380e-04, -0.00000000e+00,\n",
              "               -2.00832752e-03, -0.00000000e+00, -0.00000000e+00,\n",
              "                0.00000000e+00,  0.00000000e+00, -1.18122585e-02,\n",
              "               -0.00000000e+00,  1.18982214e-02,  0.00000000e+00,\n",
              "                3.12567153e-03, -0.00000000e+00, -1.43775321e-03,\n",
              "                2.70829275e-02, -0.00000000e+00, -1.85098574e-02,\n",
              "                6.92235772e-03,  0.00000000e+00,  3.92675400e-03,\n",
              "               -1.06392633e-02,  0.00000000e+00,  0.00000000e+00,\n",
              "               -0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
              "                3.20549682e-03,  1.22841634e-03, -2.24958896e-03,\n",
              "                0.00000000e+00,  2.65796762e-02, -2.72822194e-03,\n",
              "                0.00000000e+00, -1.81002896e-02,  0.00000000e+00,\n",
              "               -4.99244686e-03, -0.00000000e+00, -0.00000000e+00,\n",
              "               -0.00000000e+00,  0.00000000e+00,  6.78644341e-04,\n",
              "                1.37490779e-03, -1.42207677e-02,  0.00000000e+00,\n",
              "               -9.20738094e-05, -8.63368995e-03,  0.00000000e+00,\n",
              "                2.72019068e-03, -0.00000000e+00, -0.00000000e+00,\n",
              "               -0.00000000e+00,  0.00000000e+00, -0.00000000e+00,\n",
              "               -4.54790145e-03, -0.00000000e+00, -0.00000000e+00,\n",
              "                1.70382252e-03,  0.00000000e+00,  3.03448620e-03,\n",
              "               -0.00000000e+00,  1.13559281e-02, -1.08497683e-02,\n",
              "                7.25446548e-03,  1.21353706e-03,  0.00000000e+00,\n",
              "                4.41088155e-03,  1.20525528e-02,  0.00000000e+00,\n",
              "               -1.33715139e-03, -0.00000000e+00,  0.00000000e+00,\n",
              "               -0.00000000e+00, -1.04063759e-02, -0.00000000e+00,\n",
              "               -0.00000000e+00, -0.00000000e+00, -1.40594058e-02,\n",
              "                0.00000000e+00,  0.00000000e+00, -5.56300487e-03,\n",
              "                0.00000000e+00, -0.00000000e+00, -0.00000000e+00,\n",
              "               -4.50327992e-03,  1.07821375e-02, -0.00000000e+00,\n",
              "               -0.00000000e+00, -2.45611928e-03,  1.12752558e-03,\n",
              "                0.00000000e+00,  0.00000000e+00, -0.00000000e+00,\n",
              "               -8.26951489e-03,  3.34169064e-03, -1.77164637e-02,\n",
              "               -0.00000000e+00, -0.00000000e+00,  0.00000000e+00,\n",
              "                0.00000000e+00,  0.00000000e+00,  1.45225767e-02,\n",
              "               -0.00000000e+00, -1.64370947e-02,  9.03943181e-03,\n",
              "                4.52115852e-03, -0.00000000e+00, -0.00000000e+00,\n",
              "                0.00000000e+00,  0.00000000e+00, -0.00000000e+00,\n",
              "                1.68235414e-02, -1.30509026e-02,  2.17792671e-03,\n",
              "               -0.00000000e+00,  0.00000000e+00, -0.00000000e+00,\n",
              "               -3.88933206e-03,  0.00000000e+00,  5.34161646e-03,\n",
              "               -1.33148041e-02, -5.87209500e-03,  0.00000000e+00,\n",
              "               -0.00000000e+00, -7.34864594e-03, -0.00000000e+00,\n",
              "                0.00000000e+00, -1.81253976e-03,  3.94329149e-03,\n",
              "                1.49123687e-02, -0.00000000e+00,  0.00000000e+00,\n",
              "                3.51763936e-03,  2.42102332e-02, -0.00000000e+00,\n",
              "                8.27717036e-03,  0.00000000e+00, -0.00000000e+00,\n",
              "               -9.59063182e-04,  7.18619581e-03, -1.48112066e-02,\n",
              "               -9.90522467e-03,  8.02327320e-03,  0.00000000e+00,\n",
              "               -2.68650940e-03,  6.01331564e-03, -0.00000000e+00,\n",
              "                0.00000000e+00, -0.00000000e+00,  1.82147902e-02,\n",
              "               -0.00000000e+00,  9.09766532e-04, -1.05338413e-02,\n",
              "               -0.00000000e+00, -1.10069923e-02,  3.94161511e-03,\n",
              "                0.00000000e+00,  0.00000000e+00,  1.23670073e-02,\n",
              "                2.08663382e-02,  0.00000000e+00,  2.28356523e-03,\n",
              "                1.73850439e-03,  2.25769496e-03, -1.53626334e-02,\n",
              "               -0.00000000e+00, -0.00000000e+00, -0.00000000e+00,\n",
              "                0.00000000e+00, -0.00000000e+00, -6.98709581e-03,\n",
              "                1.24287494e-02, -2.81567890e-02, -0.00000000e+00,\n",
              "               -0.00000000e+00, -1.78323407e-03,  5.76652400e-03,\n",
              "                1.10856686e-02,  0.00000000e+00,  0.00000000e+00,\n",
              "               -0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
              "               -0.00000000e+00,  0.00000000e+00, -0.00000000e+00,\n",
              "                1.10138208e-05, -3.72059387e-03, -0.00000000e+00,\n",
              "               -2.26863404e-03, -2.58393195e-02, -1.42424284e-02,\n",
              "                0.00000000e+00, -2.05444358e-03,  0.00000000e+00,\n",
              "               -0.00000000e+00,  1.35771250e-02,  0.00000000e+00,\n",
              "               -0.00000000e+00, -4.89593297e-03, -0.00000000e+00,\n",
              "                0.00000000e+00,  0.00000000e+00, -2.23575030e-02],            dtype=float32)),\n",
              " (DeviceArray([[ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
              "                 0.        ,  0.        ],\n",
              "               [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
              "                 0.        ,  0.        ],\n",
              "               [ 0.00310218,  0.00194044,  0.        , ...,  0.        ,\n",
              "                 0.        ,  0.01448026],\n",
              "               ...,\n",
              "               [ 0.00618855,  0.003871  ,  0.        , ...,  0.        ,\n",
              "                 0.        ,  0.02888673],\n",
              "               [-0.00426643, -0.00266869, -0.        , ..., -0.        ,\n",
              "                -0.        , -0.01991471],\n",
              "               [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
              "                 0.        ,  0.        ]], dtype=float32),\n",
              "  DeviceArray([ 0.00000000e+00, -0.00000000e+00,  5.87133784e-03,\n",
              "                0.00000000e+00, -1.17406128e-02, -0.00000000e+00,\n",
              "               -1.35187845e-04,  1.64238969e-03,  1.15471678e-02,\n",
              "               -0.00000000e+00, -6.35571545e-03, -1.76176541e-02,\n",
              "               -0.00000000e+00, -1.23231194e-03, -0.00000000e+00,\n",
              "                1.11359311e-03, -0.00000000e+00,  0.00000000e+00,\n",
              "                0.00000000e+00,  9.49614961e-03,  1.51447747e-02,\n",
              "                1.00582251e-02, -7.68919429e-03,  3.51850479e-03,\n",
              "                0.00000000e+00,  0.00000000e+00, -0.00000000e+00,\n",
              "               -1.23673910e-03,  0.00000000e+00,  2.11188942e-02,\n",
              "               -0.00000000e+00, -6.36735093e-03, -0.00000000e+00,\n",
              "                4.79382370e-03,  0.00000000e+00, -0.00000000e+00,\n",
              "               -0.00000000e+00, -0.00000000e+00,  0.00000000e+00,\n",
              "                7.92550202e-03,  2.36107293e-03, -0.00000000e+00,\n",
              "                0.00000000e+00,  0.00000000e+00,  6.31410582e-03,\n",
              "               -1.57539658e-02, -6.18697004e-03, -4.47165221e-03,\n",
              "               -3.58986063e-03,  0.00000000e+00, -0.00000000e+00,\n",
              "               -1.43684745e-02, -8.17882828e-03, -7.23498827e-03,\n",
              "               -3.35317047e-04,  6.14562014e-05,  0.00000000e+00,\n",
              "               -0.00000000e+00, -0.00000000e+00, -0.00000000e+00,\n",
              "               -0.00000000e+00,  3.50046461e-03, -1.56365242e-02,\n",
              "               -0.00000000e+00,  2.42631827e-02,  0.00000000e+00,\n",
              "                0.00000000e+00,  0.00000000e+00,  1.04976697e-02,\n",
              "               -0.00000000e+00, -0.00000000e+00,  0.00000000e+00,\n",
              "               -1.51680438e-02,  0.00000000e+00, -0.00000000e+00,\n",
              "                0.00000000e+00,  2.98152724e-03, -6.89024199e-03,\n",
              "                1.91899687e-02,  0.00000000e+00,  0.00000000e+00,\n",
              "               -3.26187955e-03, -2.57139886e-03, -5.01615647e-03,\n",
              "               -0.00000000e+00, -0.00000000e+00,  8.31632316e-03,\n",
              "               -7.81126041e-03, -0.00000000e+00, -1.33885927e-02,\n",
              "                1.78541318e-02, -0.00000000e+00, -0.00000000e+00,\n",
              "                0.00000000e+00,  3.33493296e-03,  0.00000000e+00,\n",
              "                1.97930895e-02, -0.00000000e+00,  1.34058623e-03,\n",
              "                5.18760504e-03,  6.78260531e-03, -6.46801526e-03,\n",
              "                0.00000000e+00, -0.00000000e+00,  1.81324466e-03,\n",
              "                0.00000000e+00, -3.65735032e-03,  0.00000000e+00,\n",
              "               -1.96057343e-04, -0.00000000e+00, -9.98481642e-04,\n",
              "               -0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
              "                0.00000000e+00,  9.69536416e-03,  1.89015362e-02,\n",
              "                0.00000000e+00, -3.49741057e-03,  2.06828509e-02,\n",
              "               -6.12630136e-03, -0.00000000e+00, -4.87818103e-03,\n",
              "               -1.21724894e-02,  1.51952449e-02, -2.80066929e-03,\n",
              "                9.30087583e-04,  0.00000000e+00, -0.00000000e+00,\n",
              "               -1.86369084e-02, -5.92531590e-03,  2.56958697e-03,\n",
              "                1.01469671e-02, -0.00000000e+00, -0.00000000e+00,\n",
              "                9.03912075e-03, -0.00000000e+00, -0.00000000e+00,\n",
              "               -5.87349059e-03, -0.00000000e+00, -2.67828791e-03,\n",
              "               -0.00000000e+00, -0.00000000e+00, -0.00000000e+00,\n",
              "               -3.59028252e-03, -9.11024306e-03, -0.00000000e+00,\n",
              "               -2.26584636e-03, -0.00000000e+00, -3.10448185e-02,\n",
              "                1.43869370e-02, -0.00000000e+00,  0.00000000e+00,\n",
              "                1.02687301e-03,  0.00000000e+00, -0.00000000e+00,\n",
              "                0.00000000e+00, -0.00000000e+00, -0.00000000e+00,\n",
              "               -0.00000000e+00, -7.07102148e-03,  4.32327017e-03,\n",
              "                3.35985376e-03, -1.54682680e-03, -1.59927998e-02,\n",
              "               -0.00000000e+00, -0.00000000e+00, -0.00000000e+00,\n",
              "                0.00000000e+00, -0.00000000e+00,  0.00000000e+00,\n",
              "               -0.00000000e+00, -2.48991372e-03, -2.18815207e-02,\n",
              "               -0.00000000e+00,  1.69303715e-02, -0.00000000e+00,\n",
              "                0.00000000e+00, -4.33715945e-03,  0.00000000e+00,\n",
              "                0.00000000e+00,  4.14346252e-03,  8.23365583e-04,\n",
              "                7.57251773e-03, -8.37654714e-03,  0.00000000e+00,\n",
              "                0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
              "               -3.33882775e-03, -0.00000000e+00, -9.09065548e-03,\n",
              "               -0.00000000e+00, -0.00000000e+00,  0.00000000e+00,\n",
              "               -1.68630912e-03, -0.00000000e+00,  1.44266086e-02,\n",
              "                0.00000000e+00, -5.82984358e-04, -0.00000000e+00,\n",
              "                2.41071843e-02, -0.00000000e+00, -3.74533073e-03,\n",
              "                5.13301976e-03,  0.00000000e+00,  1.29785612e-02,\n",
              "                3.73493484e-03,  1.22136911e-02, -0.00000000e+00,\n",
              "                0.00000000e+00, -0.00000000e+00, -9.57384054e-03,\n",
              "               -3.32937911e-02,  4.15013265e-03, -4.60875360e-03,\n",
              "               -0.00000000e+00, -1.91768899e-03,  0.00000000e+00,\n",
              "                0.00000000e+00, -0.00000000e+00,  1.69984612e-03,\n",
              "               -0.00000000e+00,  1.07032126e-02, -8.46727809e-04,\n",
              "               -8.35086778e-03,  0.00000000e+00,  0.00000000e+00,\n",
              "                1.65612751e-03,  0.00000000e+00,  0.00000000e+00,\n",
              "                0.00000000e+00, -1.01936236e-02, -1.66893713e-02,\n",
              "                1.45705685e-03,  9.96439066e-03,  4.52306680e-03,\n",
              "                4.53504297e-04, -1.49726972e-03,  2.22928822e-03,\n",
              "                0.00000000e+00,  9.56488028e-03, -0.00000000e+00,\n",
              "                0.00000000e+00, -0.00000000e+00,  0.00000000e+00,\n",
              "               -1.90062337e-02,  0.00000000e+00,  0.00000000e+00,\n",
              "                0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
              "               -6.97813230e-03, -3.49671789e-03, -1.17490394e-03,\n",
              "               -2.01556925e-03,  0.00000000e+00,  5.49921300e-03,\n",
              "               -0.00000000e+00, -0.00000000e+00,  0.00000000e+00,\n",
              "               -0.00000000e+00, -0.00000000e+00,  1.31231463e-02,\n",
              "                1.37471501e-03,  0.00000000e+00, -0.00000000e+00,\n",
              "                9.86945722e-03, -0.00000000e+00, -0.00000000e+00,\n",
              "                7.10870745e-03,  8.54908489e-03,  6.87903678e-03,\n",
              "                3.92604992e-03,  0.00000000e+00, -0.00000000e+00,\n",
              "                1.84345350e-03,  0.00000000e+00, -0.00000000e+00,\n",
              "               -0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
              "               -7.64999771e-04, -5.24920365e-03, -0.00000000e+00,\n",
              "                1.17127504e-02, -8.07485264e-03, -0.00000000e+00],            dtype=float32)),\n",
              " (DeviceArray([[-0.0000000e+00, -0.0000000e+00, -1.4349492e-02, ...,\n",
              "                -1.3033390e-01, -3.0960035e-01, -0.0000000e+00],\n",
              "               [ 0.0000000e+00,  0.0000000e+00,  6.8255141e-03, ...,\n",
              "                 6.1994933e-02,  1.4726524e-01,  0.0000000e+00],\n",
              "               [ 0.0000000e+00,  0.0000000e+00,  4.2442358e-04, ...,\n",
              "                 3.8549639e-03,  9.1572357e-03,  0.0000000e+00],\n",
              "               ...,\n",
              "               [ 0.0000000e+00,  0.0000000e+00,  2.1765793e-04, ...,\n",
              "                 1.9769485e-03,  4.6961224e-03,  0.0000000e+00],\n",
              "               [ 0.0000000e+00,  0.0000000e+00,  3.3064729e-03, ...,\n",
              "                 3.0032108e-02,  7.1339473e-02,  0.0000000e+00],\n",
              "               [ 0.0000000e+00,  0.0000000e+00,  5.5784988e-04, ...,\n",
              "                 5.0668516e-03,  1.2036002e-02,  0.0000000e+00]],            dtype=float32),\n",
              "  DeviceArray([-0.09201551,  0.04376832,  0.0027216 ,  0.00113245,\n",
              "                0.01115129,  0.00286428,  0.00420204,  0.00139572,\n",
              "                0.02120262,  0.00357719], dtype=float32))]"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## `jax.vmap`\n",
        "vmap can be used to automatically produce a function that is operating on a batch, rather than just a sample.  \n",
        "vmap adds an extra dimension to the function (usually the batch size) to process multiple samples at once with no change required in the function itself. Therefore:  \n",
        "\n",
        "$$f(x), \\text{where } x \\in \\mathbb{R}^{n \\times m} \\\\\n",
        "vmap(f(x)), \\text{where } x \\in \\mathbb{R}^{* \\times n \\times m}$$\n",
        "\n",
        "Let's batch our example from above."
      ],
      "metadata": {
        "id": "6oDGbjpzMnrW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "random_flattened_images = random.normal(random.PRNGKey(1), (10, 12 * 12))\n",
        "try:\n",
        "    predict(params, random_flattened_images)\n",
        "except TypeError as e:\n",
        "    print('This does not support batching')\n",
        "    print(e)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3fyDaGyViRqL",
        "outputId": "a0e552bb-ff0e-456b-9bd2-eef97cb100c1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This does not support batching\n",
            "Incompatible shapes for dot: got (288, 144) and (10, 144).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "batch_predict = jax.vmap(predict, in_axes=(None, 0))\n",
        "batch_predict(params, random_flattened_images)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8NsYOLWMk_7H",
        "outputId": "69747cdf-c072-4a3d-fa0a-d817f9e03e62"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DeviceArray([[-1.9015684 , -0.7690184 , -4.346179  , -2.9666512 ,\n",
              "              -2.9935122 , -4.502557  , -3.7568974 , -3.277829  ,\n",
              "              -4.484718  , -1.664767  ],\n",
              "             [-2.4899907 , -1.3559425 , -3.694668  , -2.671898  ,\n",
              "              -0.74987197, -5.108794  , -3.279263  , -5.8442473 ,\n",
              "              -3.9866915 , -3.5818477 ],\n",
              "             [-1.430965  , -3.5687988 , -0.6901064 , -3.920475  ,\n",
              "              -2.3820539 , -3.928277  , -4.6724195 , -3.3987353 ,\n",
              "              -3.2485986 , -4.031564  ],\n",
              "             [-1.431176  , -2.502234  , -1.4176011 , -4.53675   ,\n",
              "              -2.3397255 , -4.204099  , -2.2941158 , -3.9822793 ,\n",
              "              -2.5264204 , -2.1598556 ],\n",
              "             [-0.68175745, -3.3978493 , -2.1756425 , -3.368552  ,\n",
              "              -2.4247975 , -5.186468  , -2.481913  , -3.6203594 ,\n",
              "              -4.0739903 , -2.3925104 ],\n",
              "             [-2.0613995 , -1.8863201 , -0.7786894 , -6.184161  ,\n",
              "              -1.915925  , -5.25496   , -5.0345583 , -4.1975017 ,\n",
              "              -4.4872713 , -2.5927444 ],\n",
              "             [-0.51650715, -1.6078053 , -2.9181855 , -4.0813227 ,\n",
              "              -3.5187    , -5.516083  , -2.6555486 , -4.6970887 ,\n",
              "              -5.3822174 , -4.234965  ],\n",
              "             [-3.133089  , -0.446841  , -3.732358  , -3.5613067 ,\n",
              "              -2.2938037 , -5.0043583 , -4.2895226 , -3.1713321 ,\n",
              "              -2.7668219 , -3.2607865 ],\n",
              "             [-1.8096611 , -2.7540545 , -3.32295   , -4.034691  ,\n",
              "              -0.5355315 , -5.4170227 , -2.9199731 , -2.891944  ,\n",
              "              -5.169351  , -4.271878  ],\n",
              "             [-1.3848741 , -0.951226  , -4.376122  , -3.2157273 ,\n",
              "              -1.9910852 , -5.887935  , -2.5057027 , -5.7534313 ,\n",
              "              -4.8753614 , -2.5382767 ]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## `jax.pmap`\n",
        "\n",
        "To parallelize the computation across devices we can use `pmap`. It is analogous with `vmap`, just that it spreads the batch across devices. Let's take a look at the sample above running on multiple TPU cores (GPUS work as well, Colab just provides one)."
      ],
      "metadata": {
        "id": "nRC9U1TPPGRw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# let's check for devices/TPU cores\n",
        "jax.devices()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IhRM13NfQx9t",
        "outputId": "ee1b7d4f-cf7c-4bb6-dd08-8b316c1acb09"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[TpuDevice(id=0, process_index=0, coords=(0,0,0), core_on_chip=0),\n",
              " TpuDevice(id=1, process_index=0, coords=(0,0,0), core_on_chip=1),\n",
              " TpuDevice(id=2, process_index=0, coords=(1,0,0), core_on_chip=0),\n",
              " TpuDevice(id=3, process_index=0, coords=(1,0,0), core_on_chip=1),\n",
              " TpuDevice(id=4, process_index=0, coords=(0,1,0), core_on_chip=0),\n",
              " TpuDevice(id=5, process_index=0, coords=(0,1,0), core_on_chip=1),\n",
              " TpuDevice(id=6, process_index=0, coords=(1,1,0), core_on_chip=0),\n",
              " TpuDevice(id=7, process_index=0, coords=(1,1,0), core_on_chip=1)]"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# let's set the var for the number of devices available\n",
        "n_devices = jax.local_device_count()\n",
        "n_devices"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OvNM1UXyQ7ri",
        "outputId": "323b95f9-c688-47f2-c487-8b80322bbbdd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "images_for_devices = random.normal(random.PRNGKey(1), (n_devices, 12 * 12))\n",
        "\n",
        "# lets replicate weights\n",
        "replicate_weights = lambda x: jnp.broadcast_to(x, (n_devices, ) + x.shape)\n",
        "params_for_devices = jax.tree_map(replicate_weights, params)\n",
        "\n",
        "jax.pmap(predict, axis_name='batch')(params_for_devices, images_for_devices)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kZI9NXTARVrY",
        "outputId": "946924f7-45d7-46d6-af10-35a89964ccdc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ShardedDeviceArray([[-2.0198154 , -1.9559671 , -1.8922449 , -1.533138  ,\n",
              "                     -2.0705414 , -5.1176844 , -1.823818  , -3.3323402 ,\n",
              "                     -4.085059  , -4.323161  ],\n",
              "                    [-2.5576596 , -1.483922  , -0.89954543, -3.9350405 ,\n",
              "                     -3.1721997 , -5.3697085 , -1.8071759 , -3.259179  ,\n",
              "                     -5.6385245 , -4.086642  ],\n",
              "                    [-2.3339736 , -2.338798  , -1.8960398 , -3.9682643 ,\n",
              "                     -2.5795531 , -2.4005284 , -2.0761194 , -3.6529276 ,\n",
              "                     -3.049448  , -1.3005831 ],\n",
              "                    [-1.6225648 , -2.0156941 , -2.833335  , -4.3311186 ,\n",
              "                     -0.54992485, -4.5912037 , -4.9597125 , -8.063543  ,\n",
              "                     -8.015085  , -5.950221  ],\n",
              "                    [-1.3082542 , -3.6305807 , -2.809523  , -2.851048  ,\n",
              "                     -2.2890854 , -1.8557265 , -2.9675677 , -2.4283717 ,\n",
              "                     -1.8484926 , -3.4933033 ],\n",
              "                    [-2.9838533 , -1.4538615 , -3.344427  , -4.4478073 ,\n",
              "                     -1.8065474 , -3.524855  , -2.111307  , -4.275355  ,\n",
              "                     -1.4058464 , -2.354629  ],\n",
              "                    [-1.022826  , -3.2074869 , -0.93867135, -4.859113  ,\n",
              "                     -3.9186742 , -4.6450653 , -2.7360828 , -3.5543208 ,\n",
              "                     -2.6699672 , -4.7151985 ],\n",
              "                    [-2.0209486 , -0.72870636, -1.5842929 , -5.826549  ,\n",
              "                     -1.9590509 , -6.10806   , -3.6879754 , -5.2186103 ,\n",
              "                     -6.421751  , -6.426897  ]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We see that we no longer get a `DeviceArray` but rather a `ShardedDeviceArray` which means we actually distributed our samples across devices.  \n",
        "Every operation in `jax.lax`, which is the module containing all underlying implementations for the operators in jax, that starts with p, is a paralellized version."
      ],
      "metadata": {
        "id": "nHUugQq8ZQb2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Transformer Language Model"
      ],
      "metadata": {
        "id": "GrjEySvxOIMY"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sud72C1WxBBD"
      },
      "outputs": [],
      "source": [
        "class Module:\n",
        "    \"\"\"Common class to store parameters for each of the modules and\n",
        "    purify functions with side effects to have none\"\"\"\n",
        "    _submodules: Dict[str, 'Module']\n",
        "    _params: Dict[str, jnp.ndarray]\n",
        "\n",
        "    def __init__(self):\n",
        "        self._params = {}\n",
        "        self._submodules = {}\n",
        "    \n",
        "    def __getattr__(self, attr_name: str):\n",
        "        if attr_name in self._params:\n",
        "            return self._params[attr_name]\n",
        "        elif attr_name in self._submodules:\n",
        "            return self._submodules[attr_name]\n",
        "        else:\n",
        "            return self.__dict__[attr_name]\n",
        "    \n",
        "    def __setattr__(self, key: str, value: Any):\n",
        "        if isinstance(value, Module):\n",
        "            self._submodules[key] = value\n",
        "        elif isinstance(value, jnp.ndarray):\n",
        "            self._params[key] = value\n",
        "        else:\n",
        "            self.__dict__[key] = value\n",
        "\n",
        "    def _clear_params(self):\n",
        "        self._params = {}\n",
        "        for submodule in self._submodules.values():\n",
        "            submodule._clear_params()\n",
        "    \n",
        "    def get_params(self) -> Mapping[str, jnp.ndarray]:\n",
        "        \"\"\"Retrieve all parameters\"\"\"\n",
        "        params = self._params.copy()\n",
        "        for submodule_name, submodule in self._submodules.items():\n",
        "            for name, value in submodule.get_params().items():\n",
        "                params[f'{submodule_name}/{name}'] = value\n",
        "        return params\n",
        "    \n",
        "    def _set_params(self, params: Mapping[str, jnp.ndarray]):\n",
        "        for name, value in params.items():\n",
        "            self._set_param(name.split('/'), value)\n",
        "\n",
        "    def _set_param(self, param_path: List[str], value: jnp.ndarray):\n",
        "        if len(param_path) == 1:\n",
        "            self._params[param_path[0]] = value\n",
        "        else:\n",
        "            self._submodules[param_path[0]]._set_param(param_path[1:], value)\n",
        "\n",
        "    def purify(self, method: Callable) -> Callable:\n",
        "        \"\"\"Takes a function and cures it from its side effects\"\"\"\n",
        "        def pure_method(params: Mapping[str, jnp.ndarray], *args):\n",
        "            self._clear_params()\n",
        "            self._set_params(params)\n",
        "            result = method(*args)\n",
        "            return result\n",
        "        return pure_method\n",
        "\n",
        "M = TypeVar('M', bound=Module)\n",
        "class ModuleList(Module, Generic[M]):\n",
        "    \"\"\"Class to store multiple modules and their parameters\"\"\"\n",
        "    _submodules: List[M]\n",
        "\n",
        "    def __init__(self, modules: List[M]):\n",
        "        super().__init__()\n",
        "        self._submodules = modules\n",
        "\n",
        "    def __getitem__(self, idx: int) -> M:\n",
        "        return self._submodules[idx]\n",
        "    \n",
        "    def _setitem__(self, key, value):\n",
        "        raise NotImplementedError\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self._submodules)\n",
        "\n",
        "    def __getattr__(self, item):\n",
        "        return self.__dict__(item)\n",
        "    \n",
        "    def __setattr__(self, key, value):\n",
        "        self.__dict__[key] = value\n",
        "\n",
        "    def _clear_params(self):\n",
        "        self._params = {}\n",
        "        for submodule in self._submodules:\n",
        "            submodule._clear_params()\n",
        "\n",
        "    def get_params(self):\n",
        "        params = self._params\n",
        "        for i, submodule in enumerate(self._submodules):\n",
        "            for name, value in submodule.get_params().items():\n",
        "                params[f'{i}/{name}'] = value\n",
        "        return params\n",
        "\n",
        "    def _set_param(self, param_path: List[str], value: jnp.ndarray):\n",
        "        self._submodules[int(param_path[0])]._set_param(param_path[1:], value)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EiSlnxO40asg"
      },
      "outputs": [],
      "source": [
        "class Embedding(Module):\n",
        "    \"\"\"Embedding Class\"\"\"\n",
        "    def __init__(self, rnd_key: jax.random.PRNGKey, n_embeddings: int, n_dim: int):\n",
        "        super().__init__()\n",
        "        self.embeddings = jax.random.normal(rnd_key, (n_embeddings, n_dim))\n",
        "    \n",
        "    def __call__(self, ids: jnp.ndarray):\n",
        "        return self.embeddings[ids, :]\n",
        "\n",
        "class PositionalEmbedding(Module):\n",
        "    \"\"\"Learned positional embeddings\"\"\"\n",
        "    def __init__(self, rnd_key: jax.random.PRNGKey, \n",
        "                 n_vocab: int, \n",
        "                 d_model: int, \n",
        "                 max_len: int = 4096):\n",
        "        \"\"\"\n",
        "        Args: \n",
        "            rnd_key - jax random key\n",
        "            n_vocab - vocab size\n",
        "            d_model - hidden dimension\n",
        "            max_len - length of positional encoding\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        self.embeddings = Embedding(rnd_key, n_vocab, d_model)\n",
        "        self.coef = 1 / d_model ** .5\n",
        "        self.encoding = jnp.zeros((max_len, d_model))\n",
        "\n",
        "    def __call__(self, x: jnp.ndarray):\n",
        "        pe = self.encoding[:x.shape[0]]\n",
        "        return self.embeddings(x) * self.coef + pe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6-lyv-wp17hm"
      },
      "outputs": [],
      "source": [
        "class Linear(Module):\n",
        "    \"\"\"Fully connected Layer\"\"\"\n",
        "    def __init__(self, rnd_key: jax.random.PRNGKey, in_features: int, out_features: int):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            rnd_key: jax random key\n",
        "            in_features: number of incoming dimensions\n",
        "            out_features: number of outgoing dimensions\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        rnd_range = 1 / in_features ** .5\n",
        "        self.weight = jax.random.uniform(rnd_key, (in_features, out_features),\n",
        "                                         minval=-rnd_range, maxval=rnd_range)\n",
        "        self.bias = jnp.zeros((out_features, ))\n",
        "\n",
        "    def __call__(self, x: jnp.ndarray):\n",
        "        return jnp.matmul(x, self.weight) + self.bias\n",
        "\n",
        "\n",
        "class LayerNorm(Module):\n",
        "    \"\"\"Layer Normalization\"\"\"\n",
        "    def __init__(self, normalized_shape: Union[Tuple[int], List[int]], \n",
        "                 eps: float = 1e-5, \n",
        "                 elementwise_affine: bool = True):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            normalized_shape - shape of hidden dimensions\n",
        "            eps - stabilizing factor\n",
        "            elementwise_affine - learn additional parameter\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        self.eps = eps \n",
        "        self.elementwise_affine = elementwise_affine\n",
        "        self.normalized_shape = tuple(normalized_shape)\n",
        "\n",
        "        if elementwise_affine:\n",
        "            self.gain = jnp.ones(normalized_shape)\n",
        "            self.bias = jnp.zeros(normalized_shape)\n",
        "    \n",
        "    def __call__(self, x: jnp.ndarray):\n",
        "        assert self.normalized_shape == x.shape[-len(self.normalized_shape):]\n",
        "        axes = [-(i+1) for i in range(len(self.normalized_shape))]\n",
        "        mean = x.mean(axis=axes, keepdims=True)\n",
        "        mean_x2 = (x ** 2).mean(axis=axes, keepdims=True)\n",
        "        variance = mean_x2 - mean ** 2\n",
        "        x_norm = (x - mean) / (variance + self.eps) ** .5\n",
        "\n",
        "        if self.elementwise_affine:\n",
        "            x_norm = self.gain * x_norm + self.bias\n",
        "        \n",
        "        return x_norm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9nZ3OAwv4zGm"
      },
      "outputs": [],
      "source": [
        "class PreSelfAttention(Module):\n",
        "    \"\"\"Creates QKV matrices\"\"\"\n",
        "    def __init__(self, rnd_key: jax.random.PRNGKey, d_model: int, heads: int, d_k: int):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            rnd_key - jax random key\n",
        "            d_model - embedding dim\n",
        "            heads - nheads\n",
        "            d_k - dimension of heads\n",
        "        \"\"\"\n",
        "\n",
        "        super().__init__()\n",
        "        self.linear = Linear(rnd_key, d_model, heads * d_k)\n",
        "        self.heads = heads\n",
        "        self.d_k = d_k\n",
        "    \n",
        "    def __call__(self, x: jnp.ndarray):\n",
        "        head_shape = x.shape[:-1]\n",
        "        x = self.linear(x)\n",
        "        x = x.reshape(*head_shape, self.heads, self.d_k)\n",
        "        return x\n",
        "\n",
        "class SelfAttention(Module):\n",
        "    \"\"\"Masked SelfAttention\"\"\"\n",
        "    def __init__(self, rnd_key: jax.random.PRNGKey, nheads: int, d_model: int):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            rnd_key - jax random key\n",
        "            nheads - number of heads\n",
        "            d_model - embedding dim\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        _, *rnd_keys = jax.random.split(rnd_key, 5)\n",
        "        self.d_k = d_model // nheads\n",
        "        self.nheads = nheads\n",
        "\n",
        "        # prepare matrices\n",
        "        self.query = PreSelfAttention(rnd_keys[0], d_model, nheads, self.d_k)\n",
        "        self.key = PreSelfAttention(rnd_keys[1], d_model, nheads, self.d_k)\n",
        "        self.value = PreSelfAttention(rnd_keys[2], d_model, nheads, self.d_k)\n",
        "\n",
        "        self.output = Linear(rnd_keys[3], d_model, d_model)\n",
        "        self.scale = 1 / self.d_k ** .5\n",
        "\n",
        "    def __call__(self, query: jnp.ndarray,\n",
        "                 key: jnp.ndarray,\n",
        "                 value: jnp.ndarray,\n",
        "                 mask: Optional[jnp.ndarray] = None):\n",
        "        seq_len = len(query)\n",
        "\n",
        "        if mask is not None:\n",
        "            assert mask.shape[0] == query.shape[0]\n",
        "            assert mask.shape[1] == key.shape[0]\n",
        "            mask = mask[:, :, None]\n",
        "        \n",
        "        query = self.query(query)\n",
        "        key = self.key(key)\n",
        "        value = self.value(value)\n",
        "\n",
        "        # equivalent to jnp.matmul(query, key.T)\n",
        "        scores = jnp.einsum('ihd,jhd->ijh', query, key)\n",
        "        scores *= self.scale\n",
        "\n",
        "        if mask is not None:\n",
        "            scores = scores + (mask == 0) * float('-inf')\n",
        "        \n",
        "        attn = jax.nn.softmax(scores, axis=1)\n",
        "\n",
        "        # equivalent to jnp.matmul(attn, values) \n",
        "        x = jnp.einsum('ijh,jhd->ihd', attn, value)\n",
        "        x = x.reshape(seq_len, -1)\n",
        "        return self.output(x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PwXexTWt70Rb"
      },
      "outputs": [],
      "source": [
        "class FeedForward(Module):\n",
        "    \"\"\"Feed Forward Block\"\"\"\n",
        "    def __init__(self, rnd_key: jax.random.PRNGKey, d_model: int, d_ff: int, activation=jax.nn.relu):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            rnd_key - jax random key\n",
        "            d_model - embedding dim\n",
        "            d_ff - projection size\n",
        "            activation - activation functions\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        _, *rnd_keys = jax.random.split(rnd_key, 5)\n",
        "        self.layer1 = Linear(rnd_keys[0], d_model, d_ff)\n",
        "        self.layer2 = Linear(rnd_keys[1], d_ff, d_model)\n",
        "        self.activation = activation\n",
        "\n",
        "    def __call__(self, x: jnp.ndarray):\n",
        "        x = self.activation(self.layer1(x))\n",
        "        return self.layer2(x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H6OsM6q68WpG"
      },
      "outputs": [],
      "source": [
        "class TransformerBlock(Module):\n",
        "    \"\"\"Transformer Block\"\"\"\n",
        "    def __init__(self, d_model: int, self_attn: SelfAttention, feed_forward: FeedForward):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            d_model - embedding dim\n",
        "            self_attn - attention module\n",
        "            feed_forward - feed forward module\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        self.size = d_model\n",
        "        self.attn = self_attn\n",
        "        self.feed_forward = feed_forward\n",
        "        self.norm_attn = LayerNorm([d_model])\n",
        "        self.norm_ff = LayerNorm([d_model])\n",
        "\n",
        "    def __call__(self, x: jnp.ndarray, mask: jnp.ndarray):\n",
        "        z = self.norm_attn(x)\n",
        "        self_attn = self.attn(query=z,\n",
        "                              key=z,\n",
        "                              value=z,\n",
        "                              mask=mask)\n",
        "        # residual\n",
        "        x = x + self_attn\n",
        "        z = self.norm_ff(x)\n",
        "        ff = self.feed_forward(z)\n",
        "        x = x + ff\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xkSGlzFQ9bTP"
      },
      "outputs": [],
      "source": [
        "class CrossEntropyLoss(Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self._loss_vmap = jax.vmap(self._loss, in_axes=(0, 0,))\n",
        "\n",
        "    def _loss(self, output: jnp.ndarray, tgt: jnp.ndarray):\n",
        "        return -jax.nn.log_softmax(output)[tgt]\n",
        "    \n",
        "    def __call__(self, output: jnp.ndarray, tgt: jnp.ndarray):\n",
        "        return self._loss_vmap(output, tgt).mean()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jc1DR9Jr96f3"
      },
      "outputs": [],
      "source": [
        "class LanguageModelTransformer(Module):\n",
        "    \"\"\"Language Model\"\"\"\n",
        "    layers: ModuleList[TransformerBlock]\n",
        "\n",
        "    def __init__(self, rnd_key: jax.random.PRNGKey,\n",
        "                 vocab: int,\n",
        "                 d_model: int,\n",
        "                 n_layers: int,\n",
        "                 n_heads: int,\n",
        "                 d_ff: int):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            rnd_key - jax random key\n",
        "            vocab - vocab size\n",
        "            d_model - embedding dim\n",
        "            n_layers - number of `TransformerBlocks`\n",
        "            n_heads - number of attention heads\n",
        "            d_ff - feed forward projection dim\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        self.vocab = vocab\n",
        "        self.d_model = d_model\n",
        "        self.loss_func = CrossEntropyLoss()\n",
        "\n",
        "        layers = [] \n",
        "        for i in range(n_layers):\n",
        "            rnd_key, selfattn_key, ffn_key = jax.random.split(rnd_key, 3)\n",
        "            attn = SelfAttention(selfattn_key, n_heads, d_model)\n",
        "            ffn = FeedForward(ffn_key, d_model, d_ff)\n",
        "            layers.append(TransformerBlock(d_model, attn, ffn))\n",
        "\n",
        "        self.layers = ModuleList(layers)\n",
        "        rnd_key, emb_key, out_key = jax.random.split(rnd_key, 3)\n",
        "        self.embeddings = PositionalEmbedding(emb_key, vocab, d_model)\n",
        "        self.norm = LayerNorm([d_model])\n",
        "        self.output = Linear(out_key, d_model, vocab)\n",
        "\n",
        "    def __call__(self, x: jnp.ndarray):\n",
        "        seq_len = len(x)\n",
        "        # triangular mask to learn autoregressively\n",
        "        mask = jnp.tril(jnp.ones((seq_len, seq_len), bool))\n",
        "\n",
        "        x = self.embeddings(x)\n",
        "        for i, _ in enumerate(self.layers):\n",
        "            x = self.layers[i](x, mask)\n",
        "\n",
        "        return self.output(self.norm(x))\n",
        "\n",
        "    def get_loss(self, x: jnp.ndarray):\n",
        "        \"\"\"Calculate loss by using the current token predicting \n",
        "        the next token in the sequence\"\"\"\n",
        "        output = self(x)\n",
        "        return self.loss_func(output[:-1], x[1:])\n",
        "\n",
        "    def sample(self, seq: jnp.ndarray, length: int = 50):\n",
        "        \"\"\"Generate a random sequence by greedy decoding\"\"\"\n",
        "        for i in range(length):\n",
        "            idx = jnp.argmax(self(seq)[-1])\n",
        "            seq = jnp.concatenate((seq, idx[None]))\n",
        "        return seq"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c4AXLmmuAAvO"
      },
      "outputs": [],
      "source": [
        "class Dataset:\n",
        "    def __init__(self, rnd_key: jax.random.PRNGKey, seq_len: int, batch_size: int):\n",
        "        self.batch_size = batch_size\n",
        "        _, self.rnd_key = jax.random.split(rnd_key)\n",
        "\n",
        "        self.data = load_dataset('empathetic_dialogues', split='train')['utterance']\n",
        "        vocab = sorted(list(set(' '.join(self.data))))\n",
        "        self.size = len(vocab)\n",
        "\n",
        "        # create character-based vocab\n",
        "        self.stoi = {t: i for i, t in enumerate(vocab)}\n",
        "        self.itos = vocab\n",
        "        \n",
        "        samples = jnp.array([self.stoi[c] for sentence in self.data for c in sentence])  \n",
        "        self.n_batches = len(samples) // (seq_len * batch_size)\n",
        "        data = samples[:self.n_batches * seq_len * batch_size]\n",
        "        self.data = data.reshape((-1, seq_len))\n",
        "        self.idx = jnp.arange(len(self.data))\n",
        "\n",
        "    def __iter__(self):\n",
        "        self._iter_idx = 0\n",
        "        self.rnd_key, rnd_key = jax.random.split(self.rnd_key)\n",
        "        self.idx = jax.random.permutation(rnd_key, self.idx)\n",
        "        return self\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.n_batches\n",
        "\n",
        "    def __next__(self):\n",
        "        if self._iter_idx >= self.n_batches:\n",
        "            raise StopIteration()\n",
        "        \n",
        "        idx = self.idx[self._iter_idx * self.batch_size: (self._iter_idx + 1) * self.batch_size]\n",
        "        self._iter_idx += 1\n",
        "        return self.data[idx]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185,
          "referenced_widgets": [
            "9cdbae15af5f4c6091e5c497ff5d005b",
            "cac518053ef64a96906695719dd02de3",
            "0c001d1ec8d646a4b486a2ccb5b262b9",
            "1240f9e07f794fde89ab37910fc39f6b",
            "dd35da8debaa468ca54e70171454c997",
            "be9e56c5f9e94f0996a62a8436c26c6d",
            "a9398a7cb6444e0eab3b453561abbd03",
            "0b03cc07f7f346ba90159b08e978a2dd",
            "e732ae3d548142c4b88658cdde3a83e9",
            "6d0e07cdb7ad468fbd9e91b6b2f4bd2b",
            "95c25915859e4271af1b652aa4d008ab",
            "713ac9cee5864043bc6ad5a789b38211",
            "17fae130d94146de8c8de1cbbe2892b7",
            "6fbc9df1efae40d280f9b4b8f1d52a51",
            "712d51dfe910452488b5b5fc6f1a0a07",
            "7ea5445450ac4afd82f5c22b9e780bdd",
            "83d76c76578e4e53826651d6b4df2c71",
            "b6bcce5151a845c4b9d6e66154c3b6cf",
            "1522f2527ac14377a8758d2a56e92677",
            "c214d15f00f84790881780e63668ad0a",
            "867f604aa7044a91a4635dcc6786fcf6",
            "82ceed4aa0ae49b3a006f14c0f86c406",
            "49f3378f43a14a259ac5b6256087cdc9",
            "cd367d90b1e545e19e5749f551e3f020",
            "69de01b576434b11a3782342edac1b5e",
            "8cb3a860730348c1aa3f812b04c35e55",
            "394ba2ff656341b6a197b01647f20d2f",
            "f31860d6a761472da32738616ea9443a",
            "252b78e1e4d74ab7bc794ce7d6127397",
            "77e406d388504606a741e9c7d8f99bf0",
            "1bb54fc325684942abd09dba95fcb5e8",
            "2c2b55ef8bfc4e94ad7a295334fc0d03",
            "f683f7d1cb5140b6b756d88512911188",
            "fbf00323eb05449181bd17171d4c06f1",
            "a556e87abb674f00a38bc9ae203e35fa",
            "a8bb928eff50446481684cc6b520e03d",
            "fff7cbf06c644256a10cd5300bd11192",
            "1a636340a9114f35a533d2b3c2f9e7c0",
            "091b24d818ea47b28c26db1f22953ded",
            "74edffa1b2314f4ea9aa212f4a8b6c93",
            "ec97fbfd2e7042d59502045c43c20181",
            "703d8e052550481d8a151b2a55ae6508",
            "f6eba9f14c4a4b949824b77e46b6f4f3",
            "7862f4dc7f274483a8589250384850de",
            "d53a5d61c640485182dda8815573494a",
            "f59b810a86804424be40ad8d25f83e88",
            "f0a98953b6604324b0c0cd1dc20313ce",
            "5ae84584e0704ff3beb0126bcb22f13c",
            "80669f5f859841e186c88e9b08ad62a1",
            "ef6817b65b764d308263093a9fb58292",
            "409d4e2670a3407388d01d9df7974b9c",
            "fb130dc5c84d4bf2998314e81bd31278",
            "a6746ec5ee854badb153589f561384ce",
            "2002007ed1594c229863ed099a8f23c2",
            "4ca5721811e64fc4ac7d67ee43c87b11",
            "81ee21112fa642b2885efd433426f97d",
            "93b451552ad5489f9d3e94f7917ebe53",
            "7f1d11b609d043e081fbeb2c513d4b75",
            "61ac2a195b0b4a59a6baf5b5d7c98e35",
            "d6fe82c82cb64ca6ac79b756e4a7afde",
            "4acac360e72246a3832ced07c58374b4",
            "db8d3ae3254a4e6ca20d66336cb60b55",
            "e25430587a56413bbc85fb2e95af39a0",
            "3d3af0f3fa8a4f17a67f25ba639db404",
            "865ee1bad5c54d12988cc7ba4b6eb71b",
            "0f18ff8c17ca438287d5375a7de98d9f"
          ]
        },
        "id": "kNybkDnsOjXg",
        "outputId": "e80e2b33-de9f-43bf-f48e-76cba3a5f654"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/1.38k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9cdbae15af5f4c6091e5c497ff5d005b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading metadata:   0%|          | 0.00/750 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "713ac9cee5864043bc6ad5a789b38211"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Using custom data configuration default\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset empathetic_dialogues/default (download: 26.72 MiB, generated: 23.97 MiB, post-processed: Unknown size, total: 50.69 MiB) to /root/.cache/huggingface/datasets/empathetic_dialogues/default/0.1.0/09bbeed3882a67db98c73952fb3c1c9a85af83dc78f81454c2454382fd03f6cf...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/28.0M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "49f3378f43a14a259ac5b6256087cdc9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/76673 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fbf00323eb05449181bd17171d4c06f1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating validation split:   0%|          | 0/12030 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d53a5d61c640485182dda8815573494a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split:   0%|          | 0/10943 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "81ee21112fa642b2885efd433426f97d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset empathetic_dialogues downloaded and prepared to /root/.cache/huggingface/datasets/empathetic_dialogues/default/0.1.0/09bbeed3882a67db98c73952fb3c1c9a85af83dc78f81454c2454382fd03f6cf. Subsequent calls will reuse this data.\n"
          ]
        }
      ],
      "source": [
        "rnd_key = jax.random.PRNGKey(42)\n",
        "dataset = Dataset(rnd_key, seq_len=32, batch_size=128)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OFrBEmZkSZyr"
      },
      "outputs": [],
      "source": [
        "model = LanguageModelTransformer(rnd_key, dataset.size, d_model=128, n_layers=3, n_heads=8, d_ff=512)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z6w4fcZNPDOe"
      },
      "outputs": [],
      "source": [
        "params = model.get_params()\n",
        "\n",
        "# purify, vmap and jit functions\n",
        "pure_sample_fn = jax.jit(model.purify(model.sample))\n",
        "pure_forward_fn = jax.jit(jax.vmap(model.purify(model.__call__), in_axes=(None, 0), out_axes=0))\n",
        "pure_loss_fn = jax.jit(jax.vmap(model.purify(model.get_loss), in_axes=(None, 0), out_axes=0))\n",
        "\n",
        "# updated loss function\n",
        "def get_loss(params, seq):\n",
        "    return pure_loss_fn(params, seq).mean()\n",
        "\n",
        "grad_loss_fn = jax.jit(jax.grad(get_loss, argnums=0))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RwCjkDXXTG8v"
      },
      "outputs": [],
      "source": [
        "# optax (gradient processing for jax)\n",
        "optimizer = O.adam(1e-4)\n",
        "opt_state = optimizer.init(params)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_sample(params, prompt: str) -> str:\n",
        "    \"\"\"Generates a sample given a prompt\"\"\"\n",
        "    encoded = [dataset.stoi[c] for c in prompt]\n",
        "    sampled = pure_sample_fn(params, jnp.array(encoded))[len(encoded):]\n",
        "    sampled = ''.join([dataset.itos[i] for i in sampled])\n",
        "    sampled = sampled.replace('\\n', '\\\\n')\n",
        "    return prompt + sampled"
      ],
      "metadata": {
        "id": "DMYssJg0UGCC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I-XdsP3ITbt9",
        "outputId": "f0d0b3ff-d736-408f-bc55-28bd2ffa44f2"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 1/1657 [00:03<1:25:11,  3.09s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.4030607\n",
            "Generation: This This at and to the the the the the the the the the the\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  3%|▎         | 51/1657 [02:08<1:07:35,  2.53s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.3759272\n",
            "Generation: I I to the the the the the the the the the the the th\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  6%|▌         | 101/1657 [05:53<1:05:01,  2.51s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.283505\n",
            "Generation: I I was to the the the the the the the the the the th\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  9%|▉         | 151/1657 [07:57<1:00:40,  2.42s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.297039\n",
            "Generation: This This at and to the the the the the the the the the the\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 12%|█▏        | 201/1657 [09:58<58:25,  2.41s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.2494583\n",
            "Generation: Why Why at the the the the the the the the the the the th\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 15%|█▌        | 251/1657 [13:42<58:42,  2.51s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.2034698\n",
            "Generation: I I was to the the the the the the there the the ther\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 18%|█▊        | 301/1657 [15:46<58:25,  2.59s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.1353998\n",
            "Generation: This This at the the the the the the the the the the the th\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 21%|██        | 351/1657 [17:52<52:44,  2.42s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.1396492\n",
            "Generation: Why Why at the the the the the the the the the the the th\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 24%|██▍       | 401/1657 [19:55<51:08,  2.44s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.114935\n",
            "Generation: This This at and the the the the the the the the the the th\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 27%|██▋       | 451/1657 [22:00<51:14,  2.55s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.0355299\n",
            "Generation: Why Why that to the the the the the the the the the the t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 30%|███       | 501/1657 [24:03<48:41,  2.53s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.10059\n",
            "Generation: This This sould the a to the the the the the theare t thear\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 33%|███▎      | 551/1657 [26:09<48:24,  2.63s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  2.0322497\n",
            "Generation: I I was a to the the the the the they they arear t th\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 36%|███▋      | 601/1657 [28:12<41:48,  2.38s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.9251583\n",
            "Generation: I I was a to to the the the the the the are the the a\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 39%|███▉      | 651/1657 [30:14<41:02,  2.45s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.8984014\n",
            "Generation: Why Why do the to the to the to the the the ane the the t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 42%|████▏     | 701/1657 [32:17<40:51,  2.56s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.8901196\n",
            "Generation: This This so to ther too the they are arear the thear are t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 45%|████▌     | 751/1657 [34:19<37:03,  2.45s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.9105884\n",
            "Generation: I I have a so that a so the a so thear are areallllll\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 48%|████▊     | 801/1657 [36:24<35:59,  2.52s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.8824006\n",
            "Generation: Why Why do do the the the the the theane the the the ane \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 51%|█████▏    | 851/1657 [38:35<38:01,  2.83s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.8137612\n",
            "Generation: I I was a so that a so the some thear a theare thind \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 54%|█████▍    | 901/1657 [40:45<32:22,  2.57s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.8633877\n",
            "Generation: Why Why do do the the they and the theat ine ind it ithe \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 57%|█████▋    | 951/1657 [42:51<29:29,  2.51s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.8348589\n",
            "Generation: I I was a so a so and a the a some thind arrrre are a\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 60%|██████    | 1001/1657 [44:55<26:40,  2.44s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.7521099\n",
            "Generation: I I was a so to the to the to the t are are tthe are \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 63%|██████▎   | 1051/1657 [47:00<25:28,  2.52s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6947234\n",
            "Generation: This This so so to the to the to the the the theare ttthe t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 66%|██████▋   | 1101/1657 [49:05<22:53,  2.47s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.7495937\n",
            "Generation: This This so so so the a some the a theare thind thind ind \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 69%|██████▉   | 1151/1657 [51:09<20:43,  2.46s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.7331746\n",
            "Generation: This This so a so the a so the will theay asthe the the the\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 72%|███████▏  | 1201/1657 [53:15<19:14,  2.53s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.7883998\n",
            "Generation: This This so so there the will the washee theast t theast i\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 75%|███████▌  | 1251/1657 [55:18<16:41,  2.47s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.7617415\n",
            "Generation: This This so so the a so the a the tourearst thind indeare \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 79%|███████▊  | 1301/1657 [57:21<15:15,  2.57s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6397512\n",
            "Generation: Why Why do the the was and the the theare theare ttttthe \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 82%|████████▏ | 1351/1657 [59:28<12:48,  2.51s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.7074205\n",
            "Generation: This This so the work the the wore theand theand the the th\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 85%|████████▍ | 1401/1657 [1:01:32<10:37,  2.49s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.7043432\n",
            "Generation: I I had to my friend to my friend iest ie t t theat i\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 88%|████████▊ | 1451/1657 [1:03:39<08:47,  2.56s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5989954\n",
            "Generation: Why Why do the was and the was a gooood t t t thearke the\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 91%|█████████ | 1501/1657 [1:05:42<06:17,  2.42s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6823014\n",
            "Generation: Why Why do do the was and the was a the thea the theare t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 94%|█████████▎| 1551/1657 [1:07:47<04:29,  2.55s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6678592\n",
            "Generation: Why Why do do the do the do the do theade idesthid ide id\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 97%|█████████▋| 1601/1657 [1:09:57<02:16,  2.44s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6174314\n",
            "Generation: I I have the to say are the some arearvearvend thinea\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|█████████▉| 1651/1657 [1:12:00<00:14,  2.46s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6610218\n",
            "Generation: This This a so so so so and the will the the the theathe y \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1657/1657 [1:12:15<00:00,  2.62s/it]\n",
            "  0%|          | 1/1657 [00:02<1:10:03,  2.54s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6134192\n",
            "Generation: I I had to my friend to me so hearssse in in in ind i\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  3%|▎         | 51/1657 [02:08<1:08:18,  2.55s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6550312\n",
            "Generation: I I had to make the the was the soureee t theaneeee t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  6%|▌         | 101/1657 [04:13<1:04:03,  2.47s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6220365\n",
            "Generation: Why Why do you do the do the was to the thealllllllllle t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  9%|▉         | 151/1657 [06:18<1:04:58,  2.59s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6089038\n",
            "Generation: I I have to my friend to my friend t t t thest athest\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 12%|█▏        | 201/1657 [08:25<1:00:21,  2.49s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5718706\n",
            "Generation: Why Why do you have the was the workit th the the the the\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 15%|█▌        | 251/1657 [10:31<59:12,  2.53s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6135579\n",
            "Generation: I I have to make to me some and I hous hous hous hous\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 18%|█▊        | 301/1657 [12:38<54:44,  2.42s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5976912\n",
            "Generation: This This so so sorry the was a good t t thearrrrrrrrrrrrrr\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 21%|██        | 351/1657 [14:43<54:34,  2.51s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5539024\n",
            "Generation: This This and the was and I was so sourpearppearppeart tind\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 24%|██▍       | 401/1657 [16:47<52:35,  2.51s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6031406\n",
            "Generation: This This a so so so sorry and the weeelllllllleay t theay \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 27%|██▋       | 451/1657 [18:53<51:36,  2.57s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.537692\n",
            "Generation: This This a so so so sorry and the weeellllllllleeeee t the\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 30%|███       | 501/1657 [20:58<47:38,  2.47s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.527822\n",
            "Generation: I I have the was a good the was theanillllleat t thea\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 33%|███▎      | 551/1657 [23:02<46:20,  2.51s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.6061478\n",
            "Generation: I I have to my friend to my friend ind it t t the the\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 36%|███▋      | 601/1657 [25:07<43:22,  2.46s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5396212\n",
            "Generation: I I have to my friends a good to meallllllllllloooooo\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 39%|███▉      | 651/1657 [27:09<41:22,  2.47s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5511844\n",
            "Generation: This This so so so have the was and I thevevererevereverere\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 42%|████▏     | 701/1657 [29:15<39:44,  2.49s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5505096\n",
            "Generation: This This so so so sorry the weekend t they theeey theeeyou\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 45%|████▌     | 751/1657 [31:23<39:09,  2.59s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.4826074\n",
            "Generation: Why Why do you have to the was the wousplllallllealllllli\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 48%|████▊     | 801/1657 [33:28<36:02,  2.53s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5528356\n",
            "Generation: This This a so so so so sorry that and t thea ttthea ttthea\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 51%|█████▏    | 851/1657 [35:32<32:59,  2.46s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5693055\n",
            "Generation: Why Why do you have the was the was t t t the thee thee t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 54%|█████▍    | 901/1657 [37:41<32:07,  2.55s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.543125\n",
            "Generation: This This so so something the was a greallllllllooooond t t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 57%|█████▋    | 951/1657 [39:46<30:08,  2.56s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5611529\n",
            "Generation: This This a so so so so so so so so sclourrrrrreaifrrrrrrea\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 60%|██████    | 1001/1657 [41:50<26:40,  2.44s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.4974496\n",
            "Generation: This This so so so so so so so in the theat the theat the t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 63%|██████▎   | 1051/1657 [43:51<24:59,  2.47s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.52046\n",
            "Generation: I I had to so some to sometimes ande t the the thevea\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 66%|██████▋   | 1101/1657 [45:59<23:38,  2.55s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5620904\n",
            "Generation: This This a so so so sorry to hear theallllleaidealllllllll\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 69%|██████▉   | 1151/1657 [48:05<20:53,  2.48s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.4932652\n",
            "Generation: I I had to have a good that is a greanevea tounoundee\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 72%|███████▏  | 1201/1657 [50:10<18:37,  2.45s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.4279946\n",
            "Generation: I I have to make a bus I have to theaiseallllllllllll\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 75%|███████▌  | 1251/1657 [52:11<16:19,  2.41s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5265495\n",
            "Generation: Why Why do you have to get to the weeeereereereere thit t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 79%|███████▊  | 1301/1657 [54:17<14:14,  2.40s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.4692646\n",
            "Generation: This This sounds a lot the was a good innnnd ind innd t t t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 82%|████████▏ | 1351/1657 [56:27<14:12,  2.78s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.4429839\n",
            "Generation: I I have to be a big to see to make tttttttttthe the \n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 85%|████████▍ | 1401/1657 [58:34<10:50,  2.54s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5038458\n",
            "Generation: This This sounds and the was a lot theaispeais theallllllll\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 88%|████████▊ | 1451/1657 [1:00:39<08:28,  2.47s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.4969479\n",
            "Generation: Why Why did you do the was a great theaitheaitheallllllll\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 91%|█████████ | 1501/1657 [1:02:48<06:40,  2.57s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.4643668\n",
            "Generation: I I have to be and the was a good t t t theeende thee\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 94%|█████████▎| 1551/1657 [1:04:53<04:28,  2.53s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.5041748\n",
            "Generation: Why Why did you have a good to me andve the the the the t\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 97%|█████████▋| 1601/1657 [1:06:58<02:22,  2.54s/it]"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\tLoss:  1.4753598\n",
            "Generation: I I was so so sorry to hear the wouskeeeeeeeeelllllll\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|█████████▉| 1651/1657 [1:09:04<00:15,  2.57s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\tLoss:  1.4783299\n",
            "Generation: Why Why did you have a big to get toureais thealllllleais\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1657/1657 [1:09:19<00:00,  2.51s/it]\n"
          ]
        }
      ],
      "source": [
        "for epoch in range(0, 2):\n",
        "    with tqdm(total=len(dataset)) as pbar:\n",
        "        for i, data in enumerate(dataset):\n",
        "            loss = get_loss(params, data)\n",
        "            grads = grad_loss_fn(params, data)\n",
        "            updates, opt_state = optimizer.update(grads, opt_state)\n",
        "            params = O.apply_updates(params, updates)\n",
        "            pbar.update(1)\n",
        "\n",
        "            if i % 50 == 0:\n",
        "                print(f'\\tLoss: ', loss)\n",
        "                prompt = __import__('random').choice(['I', 'This', 'Why'])\n",
        "                sampled = generate_sample(params, prompt)\n",
        "                print(f'Generation: {prompt} {sampled}')\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "generate_sample(params, prompt='My bachelor thesis is')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "-DgPYA2JWC40",
        "outputId": "a0161957-132d-4fc0-910a-b615401f5b30"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'My bachelor thesis is a good the the theat theat theat theathe the the '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "***\n",
        "Now that you have a starting point to build something in JAX, all that's left to say is: Happy Building!  \n",
        "Since we built this in the pure JAX to look into the internals, you might as well also use common frameworks as FLAX, Haiku, or Trax to build your next project. It requires some switching in the mindset, but still, it is worthwhile. Let alone learning to think about how our programs are running without side effects. What you are giving up is unecessary stuff like: \"Is my tensor on the GPU?\" for \"Do I mutate anything that could mess up things?\"."
      ],
      "metadata": {
        "id": "L1p665R5yMtQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "***\n",
        "# Sources\n",
        "\n",
        "- Skye's talk about JAX: https://slideslive.com/38923687/jax-accelerated-machinelearning-research-via-composable-function-transformations-in-python\n",
        "\n",
        "- JAX Docs: https://jax.readthedocs.io/en/latest/index.html\n",
        "\n",
        "- HuggingFace x JAX/FLAX: https://www.youtube.com/watch?v=fuAyUQcVzTY | https://www.youtube.com/watch?v=__eG63ZP_5g | https://www.youtube.com/watch?v=ZCMOPkcTu3s \n",
        "\n",
        "- Awesome Jax: https://github.com/n2cholas/awesome-jax"
      ],
      "metadata": {
        "id": "mSqy4cZChpVC"
      }
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "Intro2JAX.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMpq/E83t8/v7C9QK7NStFL",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9cdbae15af5f4c6091e5c497ff5d005b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cac518053ef64a96906695719dd02de3",
              "IPY_MODEL_0c001d1ec8d646a4b486a2ccb5b262b9",
              "IPY_MODEL_1240f9e07f794fde89ab37910fc39f6b"
            ],
            "layout": "IPY_MODEL_dd35da8debaa468ca54e70171454c997"
          }
        },
        "cac518053ef64a96906695719dd02de3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be9e56c5f9e94f0996a62a8436c26c6d",
            "placeholder": "​",
            "style": "IPY_MODEL_a9398a7cb6444e0eab3b453561abbd03",
            "value": "Downloading builder script: "
          }
        },
        "0c001d1ec8d646a4b486a2ccb5b262b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b03cc07f7f346ba90159b08e978a2dd",
            "max": 1380,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e732ae3d548142c4b88658cdde3a83e9",
            "value": 1380
          }
        },
        "1240f9e07f794fde89ab37910fc39f6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d0e07cdb7ad468fbd9e91b6b2f4bd2b",
            "placeholder": "​",
            "style": "IPY_MODEL_95c25915859e4271af1b652aa4d008ab",
            "value": " 4.51k/? [00:00&lt;00:00, 72.8kB/s]"
          }
        },
        "dd35da8debaa468ca54e70171454c997": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be9e56c5f9e94f0996a62a8436c26c6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9398a7cb6444e0eab3b453561abbd03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0b03cc07f7f346ba90159b08e978a2dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e732ae3d548142c4b88658cdde3a83e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6d0e07cdb7ad468fbd9e91b6b2f4bd2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "95c25915859e4271af1b652aa4d008ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "713ac9cee5864043bc6ad5a789b38211": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_17fae130d94146de8c8de1cbbe2892b7",
              "IPY_MODEL_6fbc9df1efae40d280f9b4b8f1d52a51",
              "IPY_MODEL_712d51dfe910452488b5b5fc6f1a0a07"
            ],
            "layout": "IPY_MODEL_7ea5445450ac4afd82f5c22b9e780bdd"
          }
        },
        "17fae130d94146de8c8de1cbbe2892b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83d76c76578e4e53826651d6b4df2c71",
            "placeholder": "​",
            "style": "IPY_MODEL_b6bcce5151a845c4b9d6e66154c3b6cf",
            "value": "Downloading metadata: "
          }
        },
        "6fbc9df1efae40d280f9b4b8f1d52a51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1522f2527ac14377a8758d2a56e92677",
            "max": 750,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c214d15f00f84790881780e63668ad0a",
            "value": 750
          }
        },
        "712d51dfe910452488b5b5fc6f1a0a07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_867f604aa7044a91a4635dcc6786fcf6",
            "placeholder": "​",
            "style": "IPY_MODEL_82ceed4aa0ae49b3a006f14c0f86c406",
            "value": " 1.91k/? [00:00&lt;00:00, 22.9kB/s]"
          }
        },
        "7ea5445450ac4afd82f5c22b9e780bdd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83d76c76578e4e53826651d6b4df2c71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b6bcce5151a845c4b9d6e66154c3b6cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1522f2527ac14377a8758d2a56e92677": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c214d15f00f84790881780e63668ad0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "867f604aa7044a91a4635dcc6786fcf6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82ceed4aa0ae49b3a006f14c0f86c406": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "49f3378f43a14a259ac5b6256087cdc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cd367d90b1e545e19e5749f551e3f020",
              "IPY_MODEL_69de01b576434b11a3782342edac1b5e",
              "IPY_MODEL_8cb3a860730348c1aa3f812b04c35e55"
            ],
            "layout": "IPY_MODEL_394ba2ff656341b6a197b01647f20d2f"
          }
        },
        "cd367d90b1e545e19e5749f551e3f020": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f31860d6a761472da32738616ea9443a",
            "placeholder": "​",
            "style": "IPY_MODEL_252b78e1e4d74ab7bc794ce7d6127397",
            "value": "Downloading data: 100%"
          }
        },
        "69de01b576434b11a3782342edac1b5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77e406d388504606a741e9c7d8f99bf0",
            "max": 28022709,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1bb54fc325684942abd09dba95fcb5e8",
            "value": 28022709
          }
        },
        "8cb3a860730348c1aa3f812b04c35e55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2c2b55ef8bfc4e94ad7a295334fc0d03",
            "placeholder": "​",
            "style": "IPY_MODEL_f683f7d1cb5140b6b756d88512911188",
            "value": " 28.0M/28.0M [00:01&lt;00:00, 22.8MB/s]"
          }
        },
        "394ba2ff656341b6a197b01647f20d2f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f31860d6a761472da32738616ea9443a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "252b78e1e4d74ab7bc794ce7d6127397": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "77e406d388504606a741e9c7d8f99bf0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1bb54fc325684942abd09dba95fcb5e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2c2b55ef8bfc4e94ad7a295334fc0d03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f683f7d1cb5140b6b756d88512911188": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fbf00323eb05449181bd17171d4c06f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a556e87abb674f00a38bc9ae203e35fa",
              "IPY_MODEL_a8bb928eff50446481684cc6b520e03d",
              "IPY_MODEL_fff7cbf06c644256a10cd5300bd11192"
            ],
            "layout": "IPY_MODEL_1a636340a9114f35a533d2b3c2f9e7c0"
          }
        },
        "a556e87abb674f00a38bc9ae203e35fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_091b24d818ea47b28c26db1f22953ded",
            "placeholder": "​",
            "style": "IPY_MODEL_74edffa1b2314f4ea9aa212f4a8b6c93",
            "value": "Generating train split: 100%"
          }
        },
        "a8bb928eff50446481684cc6b520e03d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec97fbfd2e7042d59502045c43c20181",
            "max": 76673,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_703d8e052550481d8a151b2a55ae6508",
            "value": 76673
          }
        },
        "fff7cbf06c644256a10cd5300bd11192": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6eba9f14c4a4b949824b77e46b6f4f3",
            "placeholder": "​",
            "style": "IPY_MODEL_7862f4dc7f274483a8589250384850de",
            "value": " 76412/76673 [00:16&lt;00:00, 4924.71 examples/s]"
          }
        },
        "1a636340a9114f35a533d2b3c2f9e7c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "091b24d818ea47b28c26db1f22953ded": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74edffa1b2314f4ea9aa212f4a8b6c93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ec97fbfd2e7042d59502045c43c20181": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "703d8e052550481d8a151b2a55ae6508": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f6eba9f14c4a4b949824b77e46b6f4f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7862f4dc7f274483a8589250384850de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d53a5d61c640485182dda8815573494a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f59b810a86804424be40ad8d25f83e88",
              "IPY_MODEL_f0a98953b6604324b0c0cd1dc20313ce",
              "IPY_MODEL_5ae84584e0704ff3beb0126bcb22f13c"
            ],
            "layout": "IPY_MODEL_80669f5f859841e186c88e9b08ad62a1"
          }
        },
        "f59b810a86804424be40ad8d25f83e88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ef6817b65b764d308263093a9fb58292",
            "placeholder": "​",
            "style": "IPY_MODEL_409d4e2670a3407388d01d9df7974b9c",
            "value": "Generating validation split: 100%"
          }
        },
        "f0a98953b6604324b0c0cd1dc20313ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb130dc5c84d4bf2998314e81bd31278",
            "max": 12030,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a6746ec5ee854badb153589f561384ce",
            "value": 12030
          }
        },
        "5ae84584e0704ff3beb0126bcb22f13c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2002007ed1594c229863ed099a8f23c2",
            "placeholder": "​",
            "style": "IPY_MODEL_4ca5721811e64fc4ac7d67ee43c87b11",
            "value": " 12011/12030 [00:04&lt;00:00, 2702.79 examples/s]"
          }
        },
        "80669f5f859841e186c88e9b08ad62a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef6817b65b764d308263093a9fb58292": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "409d4e2670a3407388d01d9df7974b9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fb130dc5c84d4bf2998314e81bd31278": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a6746ec5ee854badb153589f561384ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2002007ed1594c229863ed099a8f23c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ca5721811e64fc4ac7d67ee43c87b11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "81ee21112fa642b2885efd433426f97d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_93b451552ad5489f9d3e94f7917ebe53",
              "IPY_MODEL_7f1d11b609d043e081fbeb2c513d4b75",
              "IPY_MODEL_61ac2a195b0b4a59a6baf5b5d7c98e35"
            ],
            "layout": "IPY_MODEL_d6fe82c82cb64ca6ac79b756e4a7afde"
          }
        },
        "93b451552ad5489f9d3e94f7917ebe53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4acac360e72246a3832ced07c58374b4",
            "placeholder": "​",
            "style": "IPY_MODEL_db8d3ae3254a4e6ca20d66336cb60b55",
            "value": "Generating test split:  99%"
          }
        },
        "7f1d11b609d043e081fbeb2c513d4b75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e25430587a56413bbc85fb2e95af39a0",
            "max": 10943,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3d3af0f3fa8a4f17a67f25ba639db404",
            "value": 10943
          }
        },
        "61ac2a195b0b4a59a6baf5b5d7c98e35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_865ee1bad5c54d12988cc7ba4b6eb71b",
            "placeholder": "​",
            "style": "IPY_MODEL_0f18ff8c17ca438287d5375a7de98d9f",
            "value": " 10839/10943 [00:04&lt;00:00, 2099.04 examples/s]"
          }
        },
        "d6fe82c82cb64ca6ac79b756e4a7afde": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4acac360e72246a3832ced07c58374b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "db8d3ae3254a4e6ca20d66336cb60b55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e25430587a56413bbc85fb2e95af39a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d3af0f3fa8a4f17a67f25ba639db404": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "865ee1bad5c54d12988cc7ba4b6eb71b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0f18ff8c17ca438287d5375a7de98d9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}