BertTokenizer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DeependraParichha1004/NLP/blob/main/BertTokenizer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "027cca7a",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "027cca7a",
        "outputId": "3e7ec48f-a09d-4853-e952-5c9bf74fa332"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 4.9 MB 5.0 MB/s \n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "! pip install -q -U \"tensorflow-text==2.8.*\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6e7de4f1",
      "metadata": {
        "id": "6e7de4f1"
      },
      "outputs": [],
      "source": [
        "! pip install -q tensorflow_datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c75d27ec",
      "metadata": {
        "id": "c75d27ec"
      },
      "outputs": [],
      "source": [
        "import tensorflow_datasets as tfds\n",
        "import tensorflow as tf\n",
        "import tensorflow_text as text\n",
        "from tensorflow_text.tools.wordpiece_vocab import bert_vocab_from_dataset as bert_vocab\n",
        "import time\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import pathlib\n",
        "import re"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1fd0ca58",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 368,
          "referenced_widgets": [
            "3dfb8850050443228cf6085913885e91",
            "289f46c469f94bc79b88690b59744814",
            "5a98505dda0a43c2a0667a8e979fab40",
            "5674731703c54327b2946839d22b7728",
            "55667f211a7043459750f6982e95c84e",
            "4dbf758e56394450be3a6e2f3859cc6b",
            "122a22d80aae4e71b2ca8faa0bcabb94",
            "55e4b022e0714985bc04af8020e9d63e",
            "f7c87decd7124032af1f58379d186f92",
            "9b96e1000bd04d138c90a3bc220702e7",
            "60572d2e2dc745558a759682e1177930",
            "85c5532bd06040ea9d26121428eb10d2",
            "2a10c96266d34b5996f9115ee5aee935",
            "995254408ee44684827fe785b01f5e86",
            "148bb684e961418c81ebc8495bab4505",
            "8ea674a62de045d39cd8b8bb7a52e58d",
            "7097f6b95533482ab5ece58dd1b2a00f",
            "b658ce4bf15f4dcf97053b912333993e",
            "073731a2bad5420980deba24285a6033",
            "afa41699663949619f879c18698374e3",
            "4592762618b8404daf1ffd21687dbaa8",
            "88f7c2028fff42d79f42784372503467",
            "9083d7aa632f48c8afadeeb9f9468b50",
            "51e8fade4b774b87a980bca6c0751e48",
            "caafa5d833bb41dbba7be94ff5a2714a",
            "9167f4b2a95e49c799988518d8d2ecbe",
            "ae44712dfc1f4eec96c7d14bb554fa81",
            "66344a0856704353ae20bee3cbea9164",
            "f96fef2f11624c8c8c43845b150582f4",
            "4a8956065cbc443d8700b2d9a4293de4",
            "92d5ab4d744b417f85412f8959824d66",
            "5a11522443c44d50ae4f5e4860b666bf",
            "d79e85f5863c4acb82cf348beb0ad114",
            "f95f43c674934f48aa4545115ef79ac9",
            "769dd0c36bca465c8bd63bb1e4a42e10",
            "4ca96817dd2341e49454abbc08723215",
            "d1fbd62865424bce95ac6de24c77d037",
            "3f718ced803e43c8892d896d8194982d",
            "0a20fe7f99e14cb48475d5466d0c99d1",
            "f4f1e3dfa17b45a7a89eaa9cfbfa723b",
            "30d12e57b70c45ab9a2703245082473a",
            "80338f8ddcab48f8adb8c62cb1eac2d5",
            "41dbae2e331040faa35611c343ccff03",
            "6ff88b1dd92f49bfaca9b36ce3508ac8",
            "3479b4c33ed845ecba220b75a3d2c1ba",
            "1a7085e627eb42df863816fed31c22d5",
            "2e38e8d1353b4f5e88c7638b0e56767a",
            "4569c85940b14c3f9cbf3db181b8e497",
            "16920c79699d492fadd35ae12501d010",
            "d07c1d9560254d10b9ed71580e107e34",
            "cb738b239859457e80e661bd401ccdf9",
            "35bd5b75c36f4f9a9feda27eec669f15",
            "6332fe5a00644978ba3565740630c3be",
            "945d32bcf11a4ad19c711d86405f3dd3",
            "fdece96bb32546fda145152b56612507",
            "5e89b8f3403341d39e06d8adfbe5712b",
            "12964045bbf447b7864f81e7e4408a25",
            "b449c8f6c7294eda828e30f87882786e",
            "02cd5782502a4535ae56e23dac49e0a2",
            "830aea096266497d9ebdfd560b46cc7d",
            "ed608489b8ba4b94835965445ed86d07",
            "ea2a48d80fa94304826f211d1c074545",
            "56df7a42fb9040b1ba7159b826cf106b",
            "6e6e95f4b66a4b1bbfea4876af95fe3c",
            "32f8acf8efda4681a32430ed441c03ad",
            "6f52ff104edd4429a4b240345b9862e5",
            "f1c0b05bdd2847f1b30e58fda0ecd781",
            "5dd81a67468b4f9cb96e77ba256407cb",
            "a9c3e4ce33864d9e860c153eac150e8f",
            "8f49df2faa6b43a8baa8b91ca8a963bb",
            "64c79e440acc47dbbfc67343366cffee",
            "e0303e1c3ca44bf9931c3f87882e2c5f",
            "84c82c664e53467ca7257c196b2e8713",
            "39918ec44bb14e8386f77fabecd228ca",
            "1c4db679f5034c28bda662cae98876ac",
            "94d49b77582e4dd29108b4588518f2c2",
            "1728273e4e574d8a88f65d6c471975da",
            "03f3dd0edb014d7aad618c14f0a23e45",
            "e91d24def65d4f919260eae8241b348e",
            "577c947153ba4f0f8f3184fe285b3f5e",
            "1d42c71080fe429683033373b7dfae95",
            "fdf49400a8a849c7b82f9ec7de41a0e6",
            "74023993ab184f34a2d48be2ee1bc0e5",
            "e726d2f5a3324b6386c2097cc04fa511",
            "b5743bb425f8459481c92e857e23a544",
            "75e2b4a26e214e70a0e7248f6b0f6109",
            "55790e5c055e401c899f3d6cf563d4e8",
            "1a8b4e79386046ecb1fc48f4a51ea877",
            "45f7fae139a74e57b4bea932c4449973",
            "41b9bd1d94fd4834b96088a2986268e1",
            "0636d8f6befb4393a7fc9fa81fe1ba73",
            "6ac043a386f04457ac6091c8947c61c7",
            "b0b45bf3c7364ee1b5e8cfe90904f4d4",
            "217d0c12659b4f9d931f906ef0606330",
            "5a90a44a2fed40038f434fc67464e71a",
            "6add2cf9cbab43be82ff5c14ea038913",
            "d9d4e6b00ac54a828aebb2f78620f212",
            "2bea25527b584d3f8c601e986d14db5e",
            "495de7d68758488da7be12dc4404663d"
          ]
        },
        "id": "1fd0ca58",
        "outputId": "2d1516ac-f465-49d6-f6b0-90545770df78"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1mDownloading and preparing dataset ted_hrlr_translate/pt_to_en/1.0.0 (download: 124.94 MiB, generated: Unknown size, total: 124.94 MiB) to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0...\u001b[0m\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Dl Completed...: 0 url [00:00, ? url/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3dfb8850050443228cf6085913885e91"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Dl Size...: 0 MiB [00:00, ? MiB/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "85c5532bd06040ea9d26121428eb10d2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Extraction completed...: 0 file [00:00, ? file/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9083d7aa632f48c8afadeeb9f9468b50"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f95f43c674934f48aa4545115ef79ac9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incompleteEP9R9Z/ted_hrlr_translate-train.tfrecord\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/51785 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3479b4c33ed845ecba220b75a3d2c1ba"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5e89b8f3403341d39e06d8adfbe5712b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incompleteEP9R9Z/ted_hrlr_translate-validation.tfrecord\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1193 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f1c0b05bdd2847f1b30e58fda0ecd781"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "03f3dd0edb014d7aad618c14f0a23e45"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incompleteEP9R9Z/ted_hrlr_translate-test.tfrecord\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1803 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "45f7fae139a74e57b4bea932c4449973"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1mDataset ted_hrlr_translate downloaded and prepared to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0. Subsequent calls will reuse this data.\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "examples,metadata=tfds.load('ted_hrlr_translate/pt_to_en',with_info=True,as_supervised=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "767ee8a5",
      "metadata": {
        "id": "767ee8a5"
      },
      "outputs": [],
      "source": [
        "train_examples,validation_examples=examples['train'],examples['validation']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d8b7bcb7",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d8b7bcb7",
        "outputId": "a153a0ca-b9a2-491f-f92d-3f0ccf2070a4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensorflow.python.data.ops.dataset_ops.PrefetchDataset"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "type(train_examples)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## English Dataset"
      ],
      "metadata": {
        "id": "IFpoXk4d6lxn"
      },
      "id": "IFpoXk4d6lxn"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6c9d0de8",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6c9d0de8",
        "outputId": "0803ad20-f84d-4e88-d8a4-b665781fc1ac"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "and when you improve searchability , you actually take away the one advantage of print , which is serendipity .\n",
            "but what if it were active ?\n"
          ]
        }
      ],
      "source": [
        "import numpy\n",
        "for pt,en in train_examples.batch(2).take(1):\n",
        "    for en1 in en.numpy():\n",
        "      print(en1.decode('utf-8'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c90051b8",
      "metadata": {
        "id": "c90051b8"
      },
      "outputs": [],
      "source": [
        "train_en=train_examples.map(lambda pt,en:en)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1030588c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1030588c",
        "outputId": "351990a0-40aa-4e30-f192-7be2d4b6efa9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<MapDataset element_spec=TensorSpec(shape=(), dtype=tf.string, name=None)>"
            ]
          },
          "metadata": {},
          "execution_count": 109
        }
      ],
      "source": [
        "train_en"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Hindi Dataset"
      ],
      "metadata": {
        "id": "T33cjkwl634N"
      },
      "id": "T33cjkwl634N"
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "data=pd.read_csv('/content/Hindi_English_Truncated_Corpus.csv')\n",
        "hin=data['hindi_sentence'].iloc[:].values\n",
        "hin=tf.data.Dataset.from_tensor_slices(hin)\n",
        "type(hin)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E--NTonURitd",
        "outputId": "ea89710b-801e-4d74-9a6e-7cc927ad3ee7"
      },
      "id": "E--NTonURitd",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensorflow.python.data.ops.dataset_ops.TensorSliceDataset"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_=data['hindi_sentence']"
      ],
      "metadata": {
        "id": "Do27ijTevdyq"
      },
      "id": "Do27ijTevdyq",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gIfwvdMfSqn-",
        "outputId": "9d894f74-3b3a-4fb0-fb87-0959a002f077"
      },
      "id": "gIfwvdMfSqn-",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0         राजनीतिज्ञों के पास जो कार्य करना चाहिए, वह कर...\n",
              "1         मई आपको ऐसे ही एक बच्चे के बारे में बताना चाहू...\n",
              "2          यह प्रतिशत भारत में हिन्दुओं प्रतिशत से अधिक है।\n",
              "3            हम ये नहीं कहना चाहते कि वो ध्यान नहीं दे पाते\n",
              "4               इन्हीं वेदों का अंतिम भाग उपनिषद कहलाता है।\n",
              "                                ...                        \n",
              "127602    आर्ट डेको शैली के निर्माण मैरीन ड्राइव और ओवल ...\n",
              "127603                      और अपने गालों में डाल लेते हैं।\n",
              "127604    जहां तक गंधक के अन्य उत्पादों का प्रश्न है , द...\n",
              "127605    Zरचना-प्रकिया को उसने एक पहेली में यों बांधा है .\n",
              "127606    हाल ही में उन्हें सरकारी ठेका मिला है करीब सौ ...\n",
              "Name: hindi_sentence, Length: 127607, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_[10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "GkTkkd8IptTp",
        "outputId": "8b81f156-b1b1-4e55-e438-01e91a892c30"
      },
      "id": "GkTkkd8IptTp",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'कृपया यह सुनिश्चित कर लें कि आप सही फॉर्म का प्रयोग कर रहें हैं .'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 195
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def hin_dataset(ds):\n",
        "  return (\n",
        "      ds\n",
        "      .cache()\n",
        "      .prefetch(tf.data.AUTOTUNE)\n",
        "  )"
      ],
      "metadata": {
        "id": "aAdsOrqhU38T"
      },
      "id": "aAdsOrqhU38T",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hindi_examples=hin_dataset(hin)\n",
        "hindi_examples"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mw1uo9mLVbZu",
        "outputId": "aa0116cd-d083-4ea1-8fe1-945e30621db8"
      },
      "id": "mw1uo9mLVbZu",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<PrefetchDataset element_spec=TensorSpec(shape=(), dtype=tf.string, name=None)>"
            ]
          },
          "metadata": {},
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for ex in hindi_examples.batch(2).take(1):\n",
        "  for ex1 in ex.numpy():\n",
        "    print(ex1.decode('utf-8'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cFELy8Sb4G6U",
        "outputId": "ed1d5291-7b68-44dd-95f2-917fea9ff7f1"
      },
      "id": "cFELy8Sb4G6U",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "राजनीतिज्ञों के पास जो कार्य करना चाहिए, वह करने कि अनुमति नहीं है .\n",
            "मई आपको ऐसे ही एक बच्चे के बारे में बताना चाहूंगी,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## English and Hindi Vocab"
      ],
      "metadata": {
        "id": "RkqjSHzK7b_b"
      },
      "id": "RkqjSHzK7b_b"
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Hindi Vocab"
      ],
      "metadata": {
        "id": "GIgySI6a7L8P"
      },
      "id": "GIgySI6a7L8P"
    },
    {
      "cell_type": "code",
      "source": [
        "bert_tokenizer_params=dict(lower_case=True)\n",
        "reserved_tokens=[\"[PAD]\", \"[UNK]\", \"[START]\", \"[END]\"]"
      ],
      "metadata": {
        "id": "P_rgxGy-D1dU"
      },
      "id": "P_rgxGy-D1dU",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bert_vocab_args = dict(\n",
        "    # The target vocabulary size\n",
        "    vocab_size = 8000,\n",
        "    # Reserved tokens that must be included in the vocabulary\n",
        "    reserved_tokens=reserved_tokens,\n",
        "    # Arguments for `text.BertTokenizer`\n",
        "    bert_tokenizer_params=bert_tokenizer_params,\n",
        "    # Arguments for `wordpiece_vocab.wordpiece_tokenizer_learner_lib.learn`\n",
        "    learn_params={},\n",
        ")"
      ],
      "metadata": {
        "id": "21kCy9yaDk31"
      },
      "id": "21kCy9yaDk31",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8147436a",
      "metadata": {
        "id": "8147436a"
      },
      "outputs": [],
      "source": [
        "def write_vocab(file_path,en_vocab):\n",
        "    with open(file_path,'w',encoding='utf-8') as f:\n",
        "        for words in en_vocab:\n",
        "            print(words,file=f)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "hindi_examples"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UFDYT25x6mZQ",
        "outputId": "7ecc8c31-e6e8-4564-fd4c-899ccdc16f74"
      },
      "id": "UFDYT25x6mZQ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<PrefetchDataset element_spec=TensorSpec(shape=(), dtype=tf.string, name=None)>"
            ]
          },
          "metadata": {},
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "hin_vocab=bert_vocab.bert_vocab_from_dataset(\n",
        "hindi_examples.batch(1000).prefetch(2),\n",
        "**bert_vocab_args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DCdJn88rVx0m",
        "outputId": "e4ff65e0-f3d3-4837-c5cf-f1134ee43757"
      },
      "id": "DCdJn88rVx0m",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 2min 39s, sys: 844 ms, total: 2min 40s\n",
            "Wall time: 2min 45s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "write_vocab('hin_vocab.txt', hin_vocab)"
      ],
      "metadata": {
        "id": "vyQUwb4eWYxk"
      },
      "id": "vyQUwb4eWYxk",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### English Vocab"
      ],
      "metadata": {
        "id": "zWMk1ywVvHke"
      },
      "id": "zWMk1ywVvHke"
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "en_vocab=bert_vocab.bert_vocab_from_dataset(\n",
        "train_en.batch(1000).prefetch(2),\n",
        "**bert_vocab_args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KYTTplTo6yNk",
        "outputId": "1687e456-630d-49c4-f3a0-bce4cc0c461b"
      },
      "id": "KYTTplTo6yNk",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 1min 41s, sys: 3.07 s, total: 1min 44s\n",
            "Wall time: 1min 40s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4fbf151d",
      "metadata": {
        "id": "4fbf151d"
      },
      "outputs": [],
      "source": [
        "write_vocab('en_vocab.txt', en_vocab)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## English and Hindi Tokenizer"
      ],
      "metadata": {
        "id": "pd9Ys9hZvMIv"
      },
      "id": "pd9Ys9hZvMIv"
    },
    {
      "cell_type": "markdown",
      "source": [
        "### English Tokenizer"
      ],
      "metadata": {
        "id": "yXADB3piEkME"
      },
      "id": "yXADB3piEkME"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dbf116b0",
      "metadata": {
        "id": "dbf116b0"
      },
      "outputs": [],
      "source": [
        "import tensorflow_text as text\n",
        "en_tokenizer=text.BertTokenizer('en_vocab.txt',**bert_tokenizer_params)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3e8f9c9b",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3e8f9c9b",
        "outputId": "1000013d-5ce4-4c62-e7ac-b3b207efc052"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[b'and when you improve searchability , you actually take away the one advantage of print , which is serendipity .'\n",
            " b'but what if it were active ?' b\"but they did n't test for curiosity .\"]\n"
          ]
        }
      ],
      "source": [
        "for en_examples in train_en.batch(3).take(1):\n",
        "    print(en_examples.numpy())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "048c147e",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "048c147e",
        "outputId": "439bfec8-36b3-4c8c-8c34-6f05798b5ac9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[[72], [117], [79], [1259], [1491, 2362], [13], [79], [150], [184], [311],\n",
              "  [71], [103], [2308], [74], [2679], [13], [148], [80],\n",
              "  [55, 4840, 1434, 2423, 540], [15]]                                       ,\n",
              " [[87], [90], [107], [76], [129], [1852], [30]],\n",
              " [[87], [83], [149], [50], [9], [56], [664], [85], [2512], [15]]]>"
            ]
          },
          "metadata": {},
          "execution_count": 252
        }
      ],
      "source": [
        "sequences_eng=en_tokenizer.tokenize(en_examples)\n",
        "sequences_eng"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f0c9c286",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f0c9c286",
        "outputId": "9b38d4d8-98b3-4f62-da43-7ade2f1afb85"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[72, 117, 79, 1259, 1491, 2362, 13, 79, 150, 184, 311, 71, 103, 2308, 74,\n",
              "  2679, 13, 148, 80, 55, 4840, 1434, 2423, 540, 15]                       ,\n",
              " [87, 90, 107, 76, 129, 1852, 30],\n",
              " [87, 83, 149, 50, 9, 56, 664, 85, 2512, 15]]>"
            ]
          },
          "metadata": {},
          "execution_count": 253
        }
      ],
      "source": [
        "sequences_eng=sequences_eng.merge_dims(-2,-1)#merge_dims->(word,word-piece)\n",
        "sequences_eng"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Words like \"searchability\" and \"serendipity\" <br>have been decomposed into \"search ##ability\" and \"s ##ere ##nd<br>##ip ##ity\" using tf.gather."
      ],
      "metadata": {
        "id": "BRljm-zk-BRW"
      },
      "id": "BRljm-zk-BRW"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a0cd5e0f",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a0cd5e0f",
        "outputId": "e8b992bb-1642-4469-929d-60fabb446f70"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[b'and', b'when', b'you', b'improve', b'search', b'##ability', b',',\n",
              "  b'you', b'actually', b'take', b'away', b'the', b'one', b'advantage',\n",
              "  b'of', b'print', b',', b'which', b'is', b's', b'##ere', b'##nd', b'##ip',\n",
              "  b'##ity', b'.']                                                          ,\n",
              " [b'but', b'what', b'if', b'it', b'were', b'active', b'?'],\n",
              " [b'but', b'they', b'did', b'n', b\"'\", b't', b'test', b'for', b'curiosity',\n",
              "  b'.']                                                                    ]>"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ],
      "source": [
        "tokens_eng=tf.gather(en_vocab,sequences_eng)\n",
        "tokens_eng"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sent_en=en_tokenizer.detokenize(sequences_eng)\n",
        "tf.strings.reduce_join(sent_en,separator=' ',axis=-1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jXOOIXxD84eI",
        "outputId": "23e10e1d-657c-4d69-acde-5fd2a1c6eab5"
      },
      "id": "jXOOIXxD84eI",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
              "array([b'and when you improve searchability , you actually take away the one advantage of print , which is serendipity .',\n",
              "       b'but what if it were active ?',\n",
              "       b\"but they did n ' t test for curiosity .\"], dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sent_en"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uMQVPpww-YFx",
        "outputId": "830fc8ce-cdd4-4dc6-9d1c-5d66a993e705"
      },
      "id": "uMQVPpww-YFx",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[b'and', b'when', b'you', b'improve', b'searchability', b',', b'you',\n",
              "  b'actually', b'take', b'away', b'the', b'one', b'advantage', b'of',\n",
              "  b'print', b',', b'which', b'is', b'serendipity', b'.']              ,\n",
              " [b'but', b'what', b'if', b'it', b'were', b'active', b'?'],\n",
              " [b'but', b'they', b'did', b'n', b\"'\", b't', b'test', b'for', b'curiosity',\n",
              "  b'.']                                                                    ]>"
            ]
          },
          "metadata": {},
          "execution_count": 130
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Hindi Tokenizer"
      ],
      "metadata": {
        "id": "vtT542rnEVFu"
      },
      "id": "vtT542rnEVFu"
    },
    {
      "cell_type": "code",
      "source": [
        "hin_tokenizer=text.BertTokenizer('hin_vocab.txt',**bert_tokenizer_params)"
      ],
      "metadata": {
        "id": "NcuogRxx8fG9"
      },
      "id": "NcuogRxx8fG9",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for hn1 in hindi_examples.batch(3).take(1):\n",
        "  print(hn1.numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uwF47pfE1rQj",
        "outputId": "7f3d923f-ade9-4bc2-a1b7-f5fe1becc435"
      },
      "id": "uwF47pfE1rQj",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[b'\\xe0\\xa4\\xb0\\xe0\\xa4\\xbe\\xe0\\xa4\\x9c\\xe0\\xa4\\xa8\\xe0\\xa5\\x80\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\x9c\\xe0\\xa5\\x8d\\xe0\\xa4\\x9e\\xe0\\xa5\\x8b\\xe0\\xa4\\x82 \\xe0\\xa4\\x95\\xe0\\xa5\\x87 \\xe0\\xa4\\xaa\\xe0\\xa4\\xbe\\xe0\\xa4\\xb8 \\xe0\\xa4\\x9c\\xe0\\xa5\\x8b \\xe0\\xa4\\x95\\xe0\\xa4\\xbe\\xe0\\xa4\\xb0\\xe0\\xa5\\x8d\\xe0\\xa4\\xaf \\xe0\\xa4\\x95\\xe0\\xa4\\xb0\\xe0\\xa4\\xa8\\xe0\\xa4\\xbe \\xe0\\xa4\\x9a\\xe0\\xa4\\xbe\\xe0\\xa4\\xb9\\xe0\\xa4\\xbf\\xe0\\xa4\\x8f, \\xe0\\xa4\\xb5\\xe0\\xa4\\xb9 \\xe0\\xa4\\x95\\xe0\\xa4\\xb0\\xe0\\xa4\\xa8\\xe0\\xa5\\x87 \\xe0\\xa4\\x95\\xe0\\xa4\\xbf \\xe0\\xa4\\x85\\xe0\\xa4\\xa8\\xe0\\xa5\\x81\\xe0\\xa4\\xae\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf \\xe0\\xa4\\xa8\\xe0\\xa4\\xb9\\xe0\\xa5\\x80\\xe0\\xa4\\x82 \\xe0\\xa4\\xb9\\xe0\\xa5\\x88 .'\n",
            " b'\\xe0\\xa4\\xae\\xe0\\xa4\\x88 \\xe0\\xa4\\x86\\xe0\\xa4\\xaa\\xe0\\xa4\\x95\\xe0\\xa5\\x8b \\xe0\\xa4\\x90\\xe0\\xa4\\xb8\\xe0\\xa5\\x87 \\xe0\\xa4\\xb9\\xe0\\xa5\\x80 \\xe0\\xa4\\x8f\\xe0\\xa4\\x95 \\xe0\\xa4\\xac\\xe0\\xa4\\x9a\\xe0\\xa5\\x8d\\xe0\\xa4\\x9a\\xe0\\xa5\\x87 \\xe0\\xa4\\x95\\xe0\\xa5\\x87 \\xe0\\xa4\\xac\\xe0\\xa4\\xbe\\xe0\\xa4\\xb0\\xe0\\xa5\\x87 \\xe0\\xa4\\xae\\xe0\\xa5\\x87\\xe0\\xa4\\x82 \\xe0\\xa4\\xac\\xe0\\xa4\\xa4\\xe0\\xa4\\xbe\\xe0\\xa4\\xa8\\xe0\\xa4\\xbe \\xe0\\xa4\\x9a\\xe0\\xa4\\xbe\\xe0\\xa4\\xb9\\xe0\\xa5\\x82\\xe0\\xa4\\x82\\xe0\\xa4\\x97\\xe0\\xa5\\x80,'\n",
            " b'\\xe0\\xa4\\xaf\\xe0\\xa4\\xb9 \\xe0\\xa4\\xaa\\xe0\\xa5\\x8d\\xe0\\xa4\\xb0\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\xb6\\xe0\\xa4\\xa4 \\xe0\\xa4\\xad\\xe0\\xa4\\xbe\\xe0\\xa4\\xb0\\xe0\\xa4\\xa4 \\xe0\\xa4\\xae\\xe0\\xa5\\x87\\xe0\\xa4\\x82 \\xe0\\xa4\\xb9\\xe0\\xa4\\xbf\\xe0\\xa4\\xa8\\xe0\\xa5\\x8d\\xe0\\xa4\\xa6\\xe0\\xa5\\x81\\xe0\\xa4\\x93\\xe0\\xa4\\x82 \\xe0\\xa4\\xaa\\xe0\\xa5\\x8d\\xe0\\xa4\\xb0\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\xb6\\xe0\\xa4\\xa4 \\xe0\\xa4\\xb8\\xe0\\xa5\\x87 \\xe0\\xa4\\x85\\xe0\\xa4\\xa7\\xe0\\xa4\\xbf\\xe0\\xa4\\x95 \\xe0\\xa4\\xb9\\xe0\\xa5\\x88\\xe0\\xa5\\xa4']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for hn1 in hindi_examples.batch(3).take(1):\n",
        "  for hin1 in hn1.numpy():\n",
        "    print(hin1.decode('utf-8'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LGNMqk-sEUau",
        "outputId": "5ef2d098-28c3-4ced-b173-0f1fb799389b"
      },
      "id": "LGNMqk-sEUau",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "राजनीतिज्ञों के पास जो कार्य करना चाहिए, वह करने कि अनुमति नहीं है .\n",
            "मई आपको ऐसे ही एक बच्चे के बारे में बताना चाहूंगी,\n",
            "यह प्रतिशत भारत में हिन्दुओं प्रतिशत से अधिक है।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sequences_hin=hin_tokenizer.tokenize(hn1)\n",
        "sequences_hin\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W-wJwsgBFM4W",
        "outputId": "256be602-3e84-41d6-c144-52756016c918"
      },
      "id": "W-wJwsgBFM4W",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[[1186, 6339], [105], [308], [205], [370], [260], [286], [15], [219],\n",
              "  [209], [194], [1049], [197], [138], [17]]                           ,\n",
              " [[1481], [361], [334], [202], [192], [535], [105], [237], [129], [2263],\n",
              "  [5997], [15]]                                                          ,\n",
              " [[196], [441], [227], [129], [1849], [441], [137], [254], [138], [150]]]>"
            ]
          },
          "metadata": {},
          "execution_count": 122
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sequences_hin=sequences_hin.merge_dims(-2,-1)\n",
        "sequences_hin"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D6BW1xkrT3Gz",
        "outputId": "2df56888-335f-490c-f3db-db49d7e160d8"
      },
      "id": "D6BW1xkrT3Gz",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[1186, 6339, 105, 308, 205, 370, 260, 286, 15, 219, 209, 194, 1049, 197,\n",
              "  138, 17]                                                               ,\n",
              " [1481, 361, 334, 202, 192, 535, 105, 237, 129, 2263, 5997, 15],\n",
              " [196, 441, 227, 129, 1849, 441, 137, 254, 138, 150]]>"
            ]
          },
          "metadata": {},
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "detokens=hin_tokenizer.detokenize(sequences_hin)\n",
        "tf.strings.reduce_join(detokens,separator='',axis=-1)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eWGatE-9x3Yu",
        "outputId": "6bba74db-34ca-438d-97cf-15809cd86337"
      },
      "id": "eWGatE-9x3Yu",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
              "array([b'\\xe0\\xa4\\xb0\\xe0\\xa4\\xbe\\xe0\\xa4\\x9c\\xe0\\xa4\\xa8\\xe0\\xa5\\x80\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\x9c\\xe0\\xa4\\x9e\\xe0\\xa5\\x8b\\xe0\\xa4\\x95\\xe0\\xa4\\xaa\\xe0\\xa4\\xbe\\xe0\\xa4\\xb8\\xe0\\xa4\\x9c\\xe0\\xa5\\x8b\\xe0\\xa4\\x95\\xe0\\xa4\\xbe\\xe0\\xa4\\xb0\\xe0\\xa4\\xaf\\xe0\\xa4\\x95\\xe0\\xa4\\xb0\\xe0\\xa4\\xa8\\xe0\\xa4\\xbe\\xe0\\xa4\\x9a\\xe0\\xa4\\xbe\\xe0\\xa4\\xb9\\xe0\\xa4\\xbf\\xe0\\xa4\\x8f,\\xe0\\xa4\\xb5\\xe0\\xa4\\xb9\\xe0\\xa4\\x95\\xe0\\xa4\\xb0\\xe0\\xa4\\xa8\\xe0\\xa4\\x95\\xe0\\xa4\\xbf\\xe0\\xa4\\x85\\xe0\\xa4\\xa8\\xe0\\xa4\\xae\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\xa8\\xe0\\xa4\\xb9\\xe0\\xa5\\x80\\xe0\\xa4\\xb9.',\n",
              "       b'\\xe0\\xa4\\xae\\xe0\\xa4\\x88\\xe0\\xa4\\x86\\xe0\\xa4\\xaa\\xe0\\xa4\\x95\\xe0\\xa5\\x8b\\xe0\\xa4\\x90\\xe0\\xa4\\xb8\\xe0\\xa4\\xb9\\xe0\\xa5\\x80\\xe0\\xa4\\x8f\\xe0\\xa4\\x95\\xe0\\xa4\\xac\\xe0\\xa4\\x9a\\xe0\\xa4\\x9a\\xe0\\xa4\\x95\\xe0\\xa4\\xac\\xe0\\xa4\\xbe\\xe0\\xa4\\xb0\\xe0\\xa4\\xae\\xe0\\xa4\\xac\\xe0\\xa4\\xa4\\xe0\\xa4\\xbe\\xe0\\xa4\\xa8\\xe0\\xa4\\xbe\\xe0\\xa4\\x9a\\xe0\\xa4\\xbe\\xe0\\xa4\\xb9\\xe0\\xa4\\x97\\xe0\\xa5\\x80,',\n",
              "       b'\\xe0\\xa4\\xaf\\xe0\\xa4\\xb9\\xe0\\xa4\\xaa\\xe0\\xa4\\xb0\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\xb6\\xe0\\xa4\\xa4\\xe0\\xa4\\xad\\xe0\\xa4\\xbe\\xe0\\xa4\\xb0\\xe0\\xa4\\xa4\\xe0\\xa4\\xae\\xe0\\xa4\\xb9\\xe0\\xa4\\xbf\\xe0\\xa4\\xa8\\xe0\\xa4\\xa6\\xe0\\xa4\\x93\\xe0\\xa4\\xaa\\xe0\\xa4\\xb0\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\xb6\\xe0\\xa4\\xa4\\xe0\\xa4\\xb8\\xe0\\xa4\\x85\\xe0\\xa4\\xa7\\xe0\\xa4\\xbf\\xe0\\xa4\\x95\\xe0\\xa4\\xb9\\xe0\\xa5\\xa4'],\n",
              "      dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 244
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "a=tf.strings.reduce_join(detokens,separator=' ',axis=-1)\n",
        "for elem in a.numpy():\n",
        "  print(elem.decode('utf-8'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n5HkjTR-3yft",
        "outputId": "0f870278-a734-46e1-b0a1-1af8cdec60b0"
      },
      "id": "n5HkjTR-3yft",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "राजनीतिजञो क पास जो कारय करना चाहिए , वह करन कि अनमति नही ह .\n",
            "मई आपको ऐस ही एक बचच क बार म बताना चाहगी ,\n",
            "यह परतिशत भारत म हिनदओ परतिशत स अधिक ह ।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for row in txt_tokens:\n",
        "  for elem in row.numpy():\n",
        "    print(elem.decode('utf-8'),separator='')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nBXLud_PTmHt",
        "outputId": "7a9c41b6-face-4bac-d746-e4e3eb600109"
      },
      "id": "nBXLud_PTmHt",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "राजनीति\n",
            "##जञो\n",
            "क\n",
            "पास\n",
            "जो\n",
            "कारय\n",
            "करना\n",
            "चाहिए\n",
            ",\n",
            "वह\n",
            "करन\n",
            "कि\n",
            "अनमति\n",
            "नही\n",
            "ह\n",
            ".\n",
            "मई\n",
            "आपको\n",
            "ऐस\n",
            "ही\n",
            "एक\n",
            "बचच\n",
            "क\n",
            "बार\n",
            "म\n",
            "बताना\n",
            "चाहगी\n",
            ",\n",
            "यह\n",
            "परतिशत\n",
            "भारत\n",
            "म\n",
            "हिनदओ\n",
            "परतिशत\n",
            "स\n",
            "अधिक\n",
            "ह\n",
            "।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "To re-assemble words from the extracted tokens,"
      ],
      "metadata": {
        "id": "_BhpO0vV_y14"
      },
      "id": "_BhpO0vV_y14"
    },
    {
      "cell_type": "code",
      "source": [
        "sent=en_tokenizer.detokenize(sequences_eng)\n",
        "tf.strings.reduce_join(sent,separator=' ',axis=-1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Woq3Z8qN_EbM",
        "outputId": "d1d7d037-c421-4475-c475-57cc4c3052de"
      },
      "id": "Woq3Z8qN_EbM",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
              "array([b'and when you improve searchability , you actually take away the one advantage of print , which is serendipity .',\n",
              "       b'but what if it were active ?',\n",
              "       b\"but they did n ' t test for curiosity .\"], dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Customization and Export"
      ],
      "metadata": {
        "id": "-RmQvZ6f9Fjf"
      },
      "id": "-RmQvZ6f9Fjf"
    },
    {
      "cell_type": "code",
      "source": [
        "START = tf.argmax(tf.constant(reserved_tokens) == \"[START]\")\n",
        "END = tf.argmax(tf.constant(reserved_tokens) == \"[END]\")\n",
        "\n",
        "def add_start_end(ragged):\n",
        "  count = ragged.bounding_shape()[0]\n",
        "  starts = tf.fill([count,1], START)\n",
        "  ends = tf.fill([count,1], END)\n",
        "  return tf.concat([starts, ragged, ends], axis=1)"
      ],
      "metadata": {
        "id": "Fpc2e-lTxJR4"
      },
      "id": "Fpc2e-lTxJR4",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "words = en_tokenizer.detokenize(add_start_end(sequences_eng))\n",
        "tf.strings.reduce_join(words, separator=' ', axis=-1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PH9sMWUpxN6e",
        "outputId": "215cb455-322f-45ed-c582-0fb4b4725263"
      },
      "id": "PH9sMWUpxN6e",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
              "array([b'[START] and when you improve searchability , you actually take away the one advantage of print , which is serendipity . [END]',\n",
              "       b'[START] but what if it were active ? [END]',\n",
              "       b\"[START] but they did n ' t test for curiosity . [END]\"],\n",
              "      dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "words"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G5YpYNBoyCzs",
        "outputId": "4f541a3d-d314-473e-b0c7-88fb7348b0b8"
      },
      "id": "G5YpYNBoyCzs",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[b'[START]', b'and', b'when', b'you', b'improve', b'searchability', b',',\n",
              "  b'you', b'actually', b'take', b'away', b'the', b'one', b'advantage',\n",
              "  b'of', b'print', b',', b'which', b'is', b'serendipity', b'.', b'[END]'] ,\n",
              " [b'[START]', b'but', b'what', b'if', b'it', b'were', b'active', b'?',\n",
              "  b'[END]']                                                           ,\n",
              " [b'[START]', b'but', b'they', b'did', b'n', b\"'\", b't', b'test', b'for',\n",
              "  b'curiosity', b'.', b'[END]']                                          ]>"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def cleanup_text(reserved_tokens, token_txt):\n",
        "  # Drop the reserved tokens, except for \"[UNK]\".\n",
        "  bad_tokens = [re.escape(tok) for tok in reserved_tokens if tok != \"[UNK]\"]\n",
        "  bad_token_re = \"|\".join(bad_tokens)\n",
        "\n",
        "  bad_cells = tf.strings.regex_full_match(token_txt, bad_token_re)\n",
        "  result = tf.ragged.boolean_mask(token_txt, ~bad_cells)\n",
        "\n",
        "  # Join them into strings.\n",
        "  result = tf.strings.reduce_join(result, separator=' ', axis=-1)\n",
        "\n",
        "  return result"
      ],
      "metadata": {
        "id": "MLNlr3V5xT7J"
      },
      "id": "MLNlr3V5xT7J",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cleanup_text(reserved_tokens, words).numpy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nGQBbePnxyVu",
        "outputId": "e40c8e04-a06a-45d8-b152-7d0b18f9348e"
      },
      "id": "nGQBbePnxyVu",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([b'and when you improve searchability , you actually take away the one advantage of print , which is serendipity .',\n",
              "       b'but what if it were active ?',\n",
              "       b\"but they did n ' t test for curiosity .\"], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sequences_eng"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kigN0JCYbSm4",
        "outputId": "78adc44d-5635-4c19-b096-0e5bd433bfa9"
      },
      "id": "kigN0JCYbSm4",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[72, 117, 79, 1259, 1491, 2362, 13, 79, 150, 184, 311, 71, 103, 2308, 74,\n",
              "  2679, 13, 148, 80, 55, 4840, 1434, 2423, 540, 15]                       ,\n",
              " [87, 90, 107, 76, 129, 1852, 30],\n",
              " [87, 83, 149, 50, 9, 56, 664, 85, 2512, 15]]>"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "words=en_tokenizer.detokenize(sequences_eng)\n",
        "tf.strings.reduce_join(words,separator=' ',axis=-1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LQGtn48ua-hm",
        "outputId": "adeac3f9-e6b4-4852-c5ed-e29487a3e939"
      },
      "id": "LQGtn48ua-hm",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
              "array([b'and when you improve searchability , you actually take away the one advantage of print , which is serendipity .',\n",
              "       b'but what if it were active ?',\n",
              "       b\"but they did n ' t test for curiosity .\"], dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sequences_hin"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P0qJjjGRbWWW",
        "outputId": "11d2d3ef-27e1-436e-f427-51884d579c38"
      },
      "id": "P0qJjjGRbWWW",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[1186, 6339, 105, 308, 205, 370, 260, 286, 15, 219, 209, 194, 1049, 197,\n",
              "  138, 17]                                                               ,\n",
              " [1481, 361, 334, 202, 192, 535, 105, 237, 129, 2263, 5997, 15],\n",
              " [196, 441, 227, 129, 1849, 441, 137, 254, 138, 150]]>"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "words_1=hin_tokenizer.detokenize(sequences_hin)\n",
        "words_1=tf.strings.reduce_join(words_1,separator=' ',axis=-1)\n",
        "words_1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0PfGHtGTbBej",
        "outputId": "e9f96187-9eb6-42bb-aee0-5f726ca161c5"
      },
      "id": "0PfGHtGTbBej",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
              "array([b'\\xe0\\xa4\\xb0\\xe0\\xa4\\xbe\\xe0\\xa4\\x9c\\xe0\\xa4\\xa8\\xe0\\xa5\\x80\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\x9c\\xe0\\xa4\\x9e\\xe0\\xa5\\x8b \\xe0\\xa4\\x95 \\xe0\\xa4\\xaa\\xe0\\xa4\\xbe\\xe0\\xa4\\xb8 \\xe0\\xa4\\x9c\\xe0\\xa5\\x8b \\xe0\\xa4\\x95\\xe0\\xa4\\xbe\\xe0\\xa4\\xb0\\xe0\\xa4\\xaf \\xe0\\xa4\\x95\\xe0\\xa4\\xb0\\xe0\\xa4\\xa8\\xe0\\xa4\\xbe \\xe0\\xa4\\x9a\\xe0\\xa4\\xbe\\xe0\\xa4\\xb9\\xe0\\xa4\\xbf\\xe0\\xa4\\x8f , \\xe0\\xa4\\xb5\\xe0\\xa4\\xb9 \\xe0\\xa4\\x95\\xe0\\xa4\\xb0\\xe0\\xa4\\xa8 \\xe0\\xa4\\x95\\xe0\\xa4\\xbf \\xe0\\xa4\\x85\\xe0\\xa4\\xa8\\xe0\\xa4\\xae\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf \\xe0\\xa4\\xa8\\xe0\\xa4\\xb9\\xe0\\xa5\\x80 \\xe0\\xa4\\xb9 .',\n",
              "       b'\\xe0\\xa4\\xae\\xe0\\xa4\\x88 \\xe0\\xa4\\x86\\xe0\\xa4\\xaa\\xe0\\xa4\\x95\\xe0\\xa5\\x8b \\xe0\\xa4\\x90\\xe0\\xa4\\xb8 \\xe0\\xa4\\xb9\\xe0\\xa5\\x80 \\xe0\\xa4\\x8f\\xe0\\xa4\\x95 \\xe0\\xa4\\xac\\xe0\\xa4\\x9a\\xe0\\xa4\\x9a \\xe0\\xa4\\x95 \\xe0\\xa4\\xac\\xe0\\xa4\\xbe\\xe0\\xa4\\xb0 \\xe0\\xa4\\xae \\xe0\\xa4\\xac\\xe0\\xa4\\xa4\\xe0\\xa4\\xbe\\xe0\\xa4\\xa8\\xe0\\xa4\\xbe \\xe0\\xa4\\x9a\\xe0\\xa4\\xbe\\xe0\\xa4\\xb9\\xe0\\xa4\\x97\\xe0\\xa5\\x80 ,',\n",
              "       b'\\xe0\\xa4\\xaf\\xe0\\xa4\\xb9 \\xe0\\xa4\\xaa\\xe0\\xa4\\xb0\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\xb6\\xe0\\xa4\\xa4 \\xe0\\xa4\\xad\\xe0\\xa4\\xbe\\xe0\\xa4\\xb0\\xe0\\xa4\\xa4 \\xe0\\xa4\\xae \\xe0\\xa4\\xb9\\xe0\\xa4\\xbf\\xe0\\xa4\\xa8\\xe0\\xa4\\xa6\\xe0\\xa4\\x93 \\xe0\\xa4\\xaa\\xe0\\xa4\\xb0\\xe0\\xa4\\xa4\\xe0\\xa4\\xbf\\xe0\\xa4\\xb6\\xe0\\xa4\\xa4 \\xe0\\xa4\\xb8 \\xe0\\xa4\\x85\\xe0\\xa4\\xa7\\xe0\\xa4\\xbf\\xe0\\xa4\\x95 \\xe0\\xa4\\xb9 \\xe0\\xa5\\xa4'],\n",
              "      dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cleanup_text(reserved_tokens,words_)"
      ],
      "metadata": {
        "id": "rzmBGvUnyyrP"
      },
      "id": "rzmBGvUnyyrP",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "191fca3b",
      "metadata": {
        "id": "191fca3b"
      },
      "outputs": [],
      "source": [
        "class CustomTokenizer(tf.Module):\n",
        "  def __init__(self, reserved_tokens, vocab_path):\n",
        "    self.tokenizer = text.BertTokenizer(vocab_path, lower_case=True)\n",
        "    self._reserved_tokens = reserved_tokens\n",
        "    self._vocab_path = tf.saved_model.Asset(vocab_path)\n",
        "\n",
        "    vocab = pathlib.Path(vocab_path).read_text().splitlines()\n",
        "    self.vocab = tf.Variable(vocab)\n",
        "\n",
        "    ## Create the signatures for export:   \n",
        "\n",
        "    # Include a tokenize signature for a batch of strings. \n",
        "    self.tokenize.get_concrete_function(\n",
        "        tf.TensorSpec(shape=[None], dtype=tf.string))\n",
        "\n",
        "    # Include `detokenize` and `lookup` signatures for:\n",
        "    #   * `Tensors` with shapes [tokens] and [batch, tokens]\n",
        "    #   * `RaggedTensors` with shape [batch, tokens]\n",
        "    self.detokenize.get_concrete_function(\n",
        "        tf.TensorSpec(shape=[None, None], dtype=tf.int64))\n",
        "    self.detokenize.get_concrete_function(\n",
        "          tf.RaggedTensorSpec(shape=[None, None], dtype=tf.int64))\n",
        "\n",
        "    self.lookup.get_concrete_function(\n",
        "        tf.TensorSpec(shape=[None, None], dtype=tf.int64))\n",
        "    self.lookup.get_concrete_function(\n",
        "          tf.RaggedTensorSpec(shape=[None, None], dtype=tf.int64))\n",
        "\n",
        "    # These `get_*` methods take no arguments\n",
        "    self.get_vocab_size.get_concrete_function()\n",
        "    self.get_vocab_path.get_concrete_function()\n",
        "    self.get_reserved_tokens.get_concrete_function()\n",
        "\n",
        "  @tf.function\n",
        "  def tokenize(self, strings):\n",
        "    enc = self.tokenizer.tokenize(strings)\n",
        "    # Merge the `word` and `word-piece` axes.\n",
        "    enc = enc.merge_dims(-2,-1)\n",
        "    enc = add_start_end(enc)\n",
        "    return enc\n",
        "\n",
        "  @tf.function\n",
        "  def detokenize(self, tokenized):\n",
        "    words = self.tokenizer.detokenize(tokenized)\n",
        "    return cleanup_text(self._reserved_tokens, words)\n",
        "\n",
        "  @tf.function\n",
        "  def lookup(self, token_ids):\n",
        "    return tf.gather(self.vocab, token_ids)\n",
        "\n",
        "  @tf.function\n",
        "  def get_vocab_size(self):\n",
        "    return tf.shape(self.vocab)[0]\n",
        "\n",
        "  @tf.function\n",
        "  def get_vocab_path(self):\n",
        "    return self._vocab_path\n",
        "\n",
        "  @tf.function\n",
        "  def get_reserved_tokens(self):\n",
        "    return tf.constant(self._reserved_tokens)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d1674901",
      "metadata": {
        "id": "d1674901"
      },
      "outputs": [],
      "source": [
        "tokenizers = tf.Module()\n",
        "tokenizers.en = CustomTokenizer(reserved_tokens, 'en_vocab.txt')\n",
        "tokenizers.hin = CustomTokenizer(reserved_tokens, 'hin_vocab.txt')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Save Model using saved_model"
      ],
      "metadata": {
        "id": "_uL5UlPfta1q"
      },
      "id": "_uL5UlPfta1q"
    },
    {
      "cell_type": "code",
      "source": [
        "model_name = 'ted_hrlr_translate_hn_en_converter'\n",
        "tf.saved_model.save(tokenizers, model_name)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e6OXXQ1oR4WN",
        "outputId": "9467290d-2de1-4f6b-8bd2-b6e6692bd326"
      },
      "id": "e6OXXQ1oR4WN",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Assets written to: ted_hrlr_translate_hn_en_converter/assets\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Assets written to: ted_hrlr_translate_hn_en_converter/assets\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "09db508c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "09db508c",
        "outputId": "eecd02bc-974f-4462-d332-4ac33e5daaa2"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7010"
            ]
          },
          "metadata": {},
          "execution_count": 167
        }
      ],
      "source": [
        "reloaded_tokenizers = tf.saved_model.load(model_name)\n",
        "reloaded_tokenizers.en.get_vocab_size().numpy()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "reloaded_tokenizers.hin.get_vocab_size().numpy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wPVQQWmLR9o3",
        "outputId": "131ddf7f-f36d-47f5-bd87-e055bf4eeb51"
      },
      "id": "wPVQQWmLR9o3",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7612"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ecf5bac4",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ecf5bac4",
        "outputId": "bf0e114a-8ea7-42b0-a1e3-2d9aa3b4f5d6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[   2, 4006, 2358,  687, 1192, 2365,    4,    3]])"
            ]
          },
          "metadata": {},
          "execution_count": 192
        }
      ],
      "source": [
        "tokens_eng = reloaded_tokenizers.en.tokenize(['Hello TensorFlow!'])\n",
        "tokens_eng.numpy()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "detokens_eng=reloaded_tokenizers.en.detokenize(tokens_eng)\n",
        "detokens_eng"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ShzkY1tCrnhn",
        "outputId": "8214b01a-8dd8-47ff-dd5a-b05ca4b1a6cc"
      },
      "id": "ShzkY1tCrnhn",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1,), dtype=string, numpy=array([b'hello tensorflow !'], dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 193
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(detokens_eng.numpy()[0].decode('utf-8'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QJ_AXj_3r_c5",
        "outputId": "1c6dea32-8f4d-47e9-f51d-1a359fc9452a"
      },
      "id": "QJ_AXj_3r_c5",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hello tensorflow !\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokens = reloaded_tokenizers.hin.tokenize(['कृपया यह सुनिश्चित कर लें कि आप सही फॉर्म का प्रयोग कर रहें हैं'])\n",
        "tokens\n"
      ],
      "metadata": {
        "id": "02r5jwENJ6M-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "73cbdf46-423f-46b2-d925-516099018788"
      },
      "id": "02r5jwENJ6M-",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[2, 2654, 196, 1789, 200, 132, 194, 222, 587, 5886, 190, 380, 200, 230,\n",
              "  138, 3]]>"
            ]
          },
          "metadata": {},
          "execution_count": 223
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text_tokens = reloaded_tokenizers.hin.lookup(tokens)\n",
        "text_tokens"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EFsF6hwenGmq",
        "outputId": "6783dacd-fd23-430b-b9e6-1ea77a3ebb00"
      },
      "id": "EFsF6hwenGmq",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[b'[START]', b'\\xe0\\xa4\\x95\\xe0\\xa4\\xaa\\xe0\\xa4\\xaf\\xe0\\xa4\\xbe',\n",
              "  b'\\xe0\\xa4\\xaf\\xe0\\xa4\\xb9',\n",
              "  b'\\xe0\\xa4\\xb8\\xe0\\xa4\\xa8\\xe0\\xa4\\xbf\\xe0\\xa4\\xb6\\xe0\\xa4\\x9a\\xe0\\xa4\\xbf\\xe0\\xa4\\xa4',\n",
              "  b'\\xe0\\xa4\\x95\\xe0\\xa4\\xb0', b'\\xe0\\xa4\\xb2',\n",
              "  b'\\xe0\\xa4\\x95\\xe0\\xa4\\xbf', b'\\xe0\\xa4\\x86\\xe0\\xa4\\xaa',\n",
              "  b'\\xe0\\xa4\\xb8\\xe0\\xa4\\xb9\\xe0\\xa5\\x80',\n",
              "  b'\\xe0\\xa4\\xab\\xe0\\xa5\\x89\\xe0\\xa4\\xb0\\xe0\\xa4\\xae',\n",
              "  b'\\xe0\\xa4\\x95\\xe0\\xa4\\xbe',\n",
              "  b'\\xe0\\xa4\\xaa\\xe0\\xa4\\xb0\\xe0\\xa4\\xaf\\xe0\\xa5\\x8b\\xe0\\xa4\\x97',\n",
              "  b'\\xe0\\xa4\\x95\\xe0\\xa4\\xb0', b'\\xe0\\xa4\\xb0\\xe0\\xa4\\xb9',\n",
              "  b'\\xe0\\xa4\\xb9', b'[END]']]>"
            ]
          },
          "metadata": {},
          "execution_count": 204
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "round_trip = reloaded_tokenizers.hin.detokenize(tokens)"
      ],
      "metadata": {
        "id": "Q2GzLeGMnIqx"
      },
      "id": "Q2GzLeGMnIqx",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "round_trip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cYGOCnovzrcg",
        "outputId": "f6fccf3c-423b-4169-f2ad-2ed03c32593c"
      },
      "id": "cYGOCnovzrcg",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1,), dtype=string, numpy=\n",
              "array([b'\\xe0\\xa4\\x95\\xe0\\xa4\\xaa\\xe0\\xa4\\xaf\\xe0\\xa4\\xbe \\xe0\\xa4\\xaf\\xe0\\xa4\\xb9 \\xe0\\xa4\\xb8\\xe0\\xa4\\xa8\\xe0\\xa4\\xbf\\xe0\\xa4\\xb6\\xe0\\xa4\\x9a\\xe0\\xa4\\xbf\\xe0\\xa4\\xa4 \\xe0\\xa4\\x95\\xe0\\xa4\\xb0 \\xe0\\xa4\\xb2 \\xe0\\xa4\\x95\\xe0\\xa4\\xbf \\xe0\\xa4\\x86\\xe0\\xa4\\xaa \\xe0\\xa4\\xb8\\xe0\\xa4\\xb9\\xe0\\xa5\\x80 \\xe0\\xa4\\xab\\xe0\\xa5\\x89\\xe0\\xa4\\xb0\\xe0\\xa4\\xae \\xe0\\xa4\\x95\\xe0\\xa4\\xbe \\xe0\\xa4\\xaa\\xe0\\xa4\\xb0\\xe0\\xa4\\xaf\\xe0\\xa5\\x8b\\xe0\\xa4\\x97 \\xe0\\xa4\\x95\\xe0\\xa4\\xb0 \\xe0\\xa4\\xb0\\xe0\\xa4\\xb9 \\xe0\\xa4\\xb9'],\n",
              "      dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 231
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "type(round_trip)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VJ5fB2mEw6tR",
        "outputId": "6e85938e-f74d-4bf3-b0cf-de8a9a54ab06"
      },
      "id": "VJ5fB2mEw6tR",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensorflow.python.framework.ops.EagerTensor"
            ]
          },
          "metadata": {},
          "execution_count": 221
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "round_trip.numpy()[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4m-Gc8JGndkm",
        "outputId": "3257a45a-89d6-454b-b3a7-b10016aed93f"
      },
      "id": "4m-Gc8JGndkm",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "b'\\xe0\\xa4\\x95\\xe0\\xa4\\xaa\\xe0\\xa4\\xaf\\xe0\\xa4\\xbe \\xe0\\xa4\\xaf\\xe0\\xa4\\xb9 \\xe0\\xa4\\xb8\\xe0\\xa4\\xa8\\xe0\\xa4\\xbf\\xe0\\xa4\\xb6\\xe0\\xa4\\x9a\\xe0\\xa4\\xbf\\xe0\\xa4\\xa4 \\xe0\\xa4\\x95\\xe0\\xa4\\xb0 \\xe0\\xa4\\xb2 \\xe0\\xa4\\x95\\xe0\\xa4\\xbf \\xe0\\xa4\\x86\\xe0\\xa4\\xaa \\xe0\\xa4\\xb8\\xe0\\xa4\\xb9\\xe0\\xa5\\x80 \\xe0\\xa4\\xab\\xe0\\xa5\\x89\\xe0\\xa4\\xb0\\xe0\\xa4\\xae \\xe0\\xa4\\x95\\xe0\\xa4\\xbe \\xe0\\xa4\\xaa\\xe0\\xa4\\xb0\\xe0\\xa4\\xaf\\xe0\\xa5\\x8b\\xe0\\xa4\\x97 \\xe0\\xa4\\x95\\xe0\\xa4\\xb0 \\xe0\\xa4\\xb0\\xe0\\xa4\\xb9 \\xe0\\xa4\\xb9'"
            ]
          },
          "metadata": {},
          "execution_count": 199
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_name_hn_en='ted_hrlr_translate_hn_en_converter'\n",
        "tokenizers_hn_en=tf.saved_model.load(model_name_hn_en)"
      ],
      "metadata": {
        "id": "Jn2_o9h9nV1S"
      },
      "id": "Jn2_o9h9nV1S",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokens_en1=tokenizers_hn_en.en.tokenize(['Hi This is an Engineer .'])\n",
        "tokens_en1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nw-vS92CPBmR",
        "outputId": "d2afa84b-6a59-4ee3-a9a2-ee3680260759"
      },
      "id": "Nw-vS92CPBmR",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[2, 2265, 81, 80, 111, 3334, 15, 3]]>"
            ]
          },
          "metadata": {},
          "execution_count": 268
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "detokens_en1=tokenizers_hn_en.en.detokenize(tokens_en1)\n",
        "detokens_en1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jfavW9fFPaib",
        "outputId": "b64ed82b-bff2-4aad-c437-930e0baa882f"
      },
      "id": "jfavW9fFPaib",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1,), dtype=string, numpy=array([b'hi this is an engineer .'], dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 269
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Hindi Sentence(Example)"
      ],
      "metadata": {
        "id": "wKMRqsb0r4Kp"
      },
      "id": "wKMRqsb0r4Kp"
    },
    {
      "cell_type": "code",
      "source": [
        "tokens_hn1=tokenizers_hn_en.hin.tokenize(['और अपने गालों में डाल लेते हैं ।'])\n",
        "tokens_hn1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ISp8pDVSPzgr",
        "outputId": "d14d3d88-5554-4422-f75a-fd0e82d86003"
      },
      "id": "ISp8pDVSPzgr",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[2, 189, 213, 107, 1024, 204, 129, 1160, 818, 138, 150, 3]]>"
            ]
          },
          "metadata": {},
          "execution_count": 273
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokens_new_hn1=tokens_hn1.merge_dims(-2,-1)\n",
        "tokens_new_hn1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sS3_ZL8YQpvw",
        "outputId": "d9ab1314-8d4f-47ad-fdca-ddd3a0da5ce8"
      },
      "id": "sS3_ZL8YQpvw",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(12,), dtype=int64, numpy=\n",
              "array([   2,  189,  213,  107, 1024,  204,  129, 1160,  818,  138,  150,\n",
              "          3])>"
            ]
          },
          "metadata": {},
          "execution_count": 277
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "detokens_hn1=tokenizers_hn_en.hin.detokenize(tokens_hn1)\n",
        "for element in detokens_hn1.numpy():\n",
        "  print(element.decode('utf-8'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BE5ApkTzPzkf",
        "outputId": "74a5cea7-510a-4d08-d353-3f7be46489a2"
      },
      "id": "BE5ApkTzPzkf",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "और अपन गालो म डाल लत ह ।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        ">                                  OR\n",
        "\n"
      ],
      "metadata": {
        "id": "S2M_Y41Csz9e"
      },
      "id": "S2M_Y41Csz9e"
    },
    {
      "cell_type": "code",
      "source": [
        "tokens = reloaded_tokenizers.hin.tokenize(['कृपया यह सुनिश्चित कर लें कि आप सही फॉर्म का प्रयोग कर रहें हैं'])\n",
        "round_trip = reloaded_tokenizers.hin.detokenize(tokens)\n"
      ],
      "metadata": {
        "id": "ZXn9CJY9tSf_"
      },
      "id": "ZXn9CJY9tSf_",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(round_trip.numpy()[0].decode('utf-8'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8teOq4jgnMmI",
        "outputId": "1f432bd3-355f-4ffd-f196-63d7d25fb93f"
      },
      "id": "8teOq4jgnMmI",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "कपया यह सनिशचित कर ल कि आप सही फॉरम का परयोग कर रह ह\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Portuguese Sentences(Example)"
      ],
      "metadata": {
        "id": "x5A1-xq_r-WM"
      },
      "id": "x5A1-xq_r-WM"
    },
    {
      "cell_type": "code",
      "source": [
        "for element in pt_examples.numpy():\n",
        "  print(element.decode('utf-8'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jWs2JJQdMBbI",
        "outputId": "e34269ce-2fd7-494f-e9ba-6867ff6be4b3"
      },
      "id": "jWs2JJQdMBbI",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "e quando melhoramos a procura , tiramos a única vantagem da impressão , que é a serendipidade .\n",
            "mas e se estes fatores fossem ativos ?\n",
            "mas eles não tinham a curiosidade de me testar .\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokens_pt=tokenizers.pt.tokenize(pt_examples)\n",
        "tokens_pt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i0Uyb6VlMQUA",
        "outputId": "1bcdeaf9-568b-4464-e4b2-cca52f6ec33a"
      },
      "id": "i0Uyb6VlMQUA",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.RaggedTensor [[2, 44, 115, 6402, 148, 40, 887, 14, 3936, 40, 463, 2715, 94, 2047, 14, 84,\n",
              "  44, 40, 117, 1328, 2721, 818, 539, 16, 3]                                 ,\n",
              " [2, 99, 44, 89, 199, 2836, 1336, 3996, 32, 3],\n",
              " [2, 99, 131, 88, 383, 40, 2673, 83, 110, 1972, 16, 3]]>"
            ]
          },
          "metadata": {},
          "execution_count": 263
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "detokens_pt=tokenizers.pt.detokenize(tokens_pt)\n",
        "detokens_pt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cyC7M-lHMbew",
        "outputId": "eca056af-345f-4446-ae19-76b76b9106b3"
      },
      "id": "cyC7M-lHMbew",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
              "array([b'e quando melhoramos a procura , tiramos a unica vantagem da impressao , que e a serendipidade .',\n",
              "       b'mas e se estes fatores fossem ativos ?',\n",
              "       b'mas eles nao tinham a curiosidade de me testar .'], dtype=object)>"
            ]
          },
          "metadata": {},
          "execution_count": 265
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    },
    "colab": {
      "name": "BertTokenizer.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3dfb8850050443228cf6085913885e91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_289f46c469f94bc79b88690b59744814",
              "IPY_MODEL_5a98505dda0a43c2a0667a8e979fab40",
              "IPY_MODEL_5674731703c54327b2946839d22b7728"
            ],
            "layout": "IPY_MODEL_55667f211a7043459750f6982e95c84e"
          }
        },
        "289f46c469f94bc79b88690b59744814": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4dbf758e56394450be3a6e2f3859cc6b",
            "placeholder": "​",
            "style": "IPY_MODEL_122a22d80aae4e71b2ca8faa0bcabb94",
            "value": "Dl Completed...: 100%"
          }
        },
        "5a98505dda0a43c2a0667a8e979fab40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_55e4b022e0714985bc04af8020e9d63e",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f7c87decd7124032af1f58379d186f92",
            "value": 1
          }
        },
        "5674731703c54327b2946839d22b7728": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9b96e1000bd04d138c90a3bc220702e7",
            "placeholder": "​",
            "style": "IPY_MODEL_60572d2e2dc745558a759682e1177930",
            "value": " 1/1 [00:16&lt;00:00,  5.31s/ url]"
          }
        },
        "55667f211a7043459750f6982e95c84e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4dbf758e56394450be3a6e2f3859cc6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "122a22d80aae4e71b2ca8faa0bcabb94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "55e4b022e0714985bc04af8020e9d63e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "f7c87decd7124032af1f58379d186f92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9b96e1000bd04d138c90a3bc220702e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60572d2e2dc745558a759682e1177930": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "85c5532bd06040ea9d26121428eb10d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2a10c96266d34b5996f9115ee5aee935",
              "IPY_MODEL_995254408ee44684827fe785b01f5e86",
              "IPY_MODEL_148bb684e961418c81ebc8495bab4505"
            ],
            "layout": "IPY_MODEL_8ea674a62de045d39cd8b8bb7a52e58d"
          }
        },
        "2a10c96266d34b5996f9115ee5aee935": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7097f6b95533482ab5ece58dd1b2a00f",
            "placeholder": "​",
            "style": "IPY_MODEL_b658ce4bf15f4dcf97053b912333993e",
            "value": "Dl Size...: 100%"
          }
        },
        "995254408ee44684827fe785b01f5e86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_073731a2bad5420980deba24285a6033",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_afa41699663949619f879c18698374e3",
            "value": 1
          }
        },
        "148bb684e961418c81ebc8495bab4505": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4592762618b8404daf1ffd21687dbaa8",
            "placeholder": "​",
            "style": "IPY_MODEL_88f7c2028fff42d79f42784372503467",
            "value": " 124/124 [00:16&lt;00:00, 25.43 MiB/s]"
          }
        },
        "8ea674a62de045d39cd8b8bb7a52e58d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7097f6b95533482ab5ece58dd1b2a00f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b658ce4bf15f4dcf97053b912333993e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "073731a2bad5420980deba24285a6033": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "afa41699663949619f879c18698374e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4592762618b8404daf1ffd21687dbaa8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88f7c2028fff42d79f42784372503467": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9083d7aa632f48c8afadeeb9f9468b50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_51e8fade4b774b87a980bca6c0751e48",
              "IPY_MODEL_caafa5d833bb41dbba7be94ff5a2714a",
              "IPY_MODEL_9167f4b2a95e49c799988518d8d2ecbe"
            ],
            "layout": "IPY_MODEL_ae44712dfc1f4eec96c7d14bb554fa81"
          }
        },
        "51e8fade4b774b87a980bca6c0751e48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66344a0856704353ae20bee3cbea9164",
            "placeholder": "​",
            "style": "IPY_MODEL_f96fef2f11624c8c8c43845b150582f4",
            "value": "Extraction completed...: 100%"
          }
        },
        "caafa5d833bb41dbba7be94ff5a2714a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a8956065cbc443d8700b2d9a4293de4",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_92d5ab4d744b417f85412f8959824d66",
            "value": 1
          }
        },
        "9167f4b2a95e49c799988518d8d2ecbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5a11522443c44d50ae4f5e4860b666bf",
            "placeholder": "​",
            "style": "IPY_MODEL_d79e85f5863c4acb82cf348beb0ad114",
            "value": " 1/1 [00:16&lt;00:00, 16.39s/ file]"
          }
        },
        "ae44712dfc1f4eec96c7d14bb554fa81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66344a0856704353ae20bee3cbea9164": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f96fef2f11624c8c8c43845b150582f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4a8956065cbc443d8700b2d9a4293de4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "92d5ab4d744b417f85412f8959824d66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5a11522443c44d50ae4f5e4860b666bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d79e85f5863c4acb82cf348beb0ad114": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f95f43c674934f48aa4545115ef79ac9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_769dd0c36bca465c8bd63bb1e4a42e10",
              "IPY_MODEL_4ca96817dd2341e49454abbc08723215",
              "IPY_MODEL_d1fbd62865424bce95ac6de24c77d037"
            ],
            "layout": "IPY_MODEL_3f718ced803e43c8892d896d8194982d"
          }
        },
        "769dd0c36bca465c8bd63bb1e4a42e10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a20fe7f99e14cb48475d5466d0c99d1",
            "placeholder": "​",
            "style": "IPY_MODEL_f4f1e3dfa17b45a7a89eaa9cfbfa723b",
            "value": ""
          }
        },
        "4ca96817dd2341e49454abbc08723215": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_30d12e57b70c45ab9a2703245082473a",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_80338f8ddcab48f8adb8c62cb1eac2d5",
            "value": 1
          }
        },
        "d1fbd62865424bce95ac6de24c77d037": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_41dbae2e331040faa35611c343ccff03",
            "placeholder": "​",
            "style": "IPY_MODEL_6ff88b1dd92f49bfaca9b36ce3508ac8",
            "value": " 51652/0 [00:21&lt;00:00, 2549.93 examples/s]"
          }
        },
        "3f718ced803e43c8892d896d8194982d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a20fe7f99e14cb48475d5466d0c99d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4f1e3dfa17b45a7a89eaa9cfbfa723b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "30d12e57b70c45ab9a2703245082473a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "80338f8ddcab48f8adb8c62cb1eac2d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "41dbae2e331040faa35611c343ccff03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ff88b1dd92f49bfaca9b36ce3508ac8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3479b4c33ed845ecba220b75a3d2c1ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1a7085e627eb42df863816fed31c22d5",
              "IPY_MODEL_2e38e8d1353b4f5e88c7638b0e56767a",
              "IPY_MODEL_4569c85940b14c3f9cbf3db181b8e497"
            ],
            "layout": "IPY_MODEL_16920c79699d492fadd35ae12501d010"
          }
        },
        "1a7085e627eb42df863816fed31c22d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d07c1d9560254d10b9ed71580e107e34",
            "placeholder": "​",
            "style": "IPY_MODEL_cb738b239859457e80e661bd401ccdf9",
            "value": "100%"
          }
        },
        "2e38e8d1353b4f5e88c7638b0e56767a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "danger",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35bd5b75c36f4f9a9feda27eec669f15",
            "max": 51785,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6332fe5a00644978ba3565740630c3be",
            "value": 51784
          }
        },
        "4569c85940b14c3f9cbf3db181b8e497": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_945d32bcf11a4ad19c711d86405f3dd3",
            "placeholder": "​",
            "style": "IPY_MODEL_fdece96bb32546fda145152b56612507",
            "value": " 51784/51785 [00:00&lt;00:00, 142667.25 examples/s]"
          }
        },
        "16920c79699d492fadd35ae12501d010": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d07c1d9560254d10b9ed71580e107e34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb738b239859457e80e661bd401ccdf9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "35bd5b75c36f4f9a9feda27eec669f15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6332fe5a00644978ba3565740630c3be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "945d32bcf11a4ad19c711d86405f3dd3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fdece96bb32546fda145152b56612507": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5e89b8f3403341d39e06d8adfbe5712b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_12964045bbf447b7864f81e7e4408a25",
              "IPY_MODEL_b449c8f6c7294eda828e30f87882786e",
              "IPY_MODEL_02cd5782502a4535ae56e23dac49e0a2"
            ],
            "layout": "IPY_MODEL_830aea096266497d9ebdfd560b46cc7d"
          }
        },
        "12964045bbf447b7864f81e7e4408a25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ed608489b8ba4b94835965445ed86d07",
            "placeholder": "​",
            "style": "IPY_MODEL_ea2a48d80fa94304826f211d1c074545",
            "value": ""
          }
        },
        "b449c8f6c7294eda828e30f87882786e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_56df7a42fb9040b1ba7159b826cf106b",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6e6e95f4b66a4b1bbfea4876af95fe3c",
            "value": 1
          }
        },
        "02cd5782502a4535ae56e23dac49e0a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_32f8acf8efda4681a32430ed441c03ad",
            "placeholder": "​",
            "style": "IPY_MODEL_6f52ff104edd4429a4b240345b9862e5",
            "value": " 992/0 [00:00&lt;00:00, 2157.05 examples/s]"
          }
        },
        "830aea096266497d9ebdfd560b46cc7d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed608489b8ba4b94835965445ed86d07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea2a48d80fa94304826f211d1c074545": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "56df7a42fb9040b1ba7159b826cf106b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "6e6e95f4b66a4b1bbfea4876af95fe3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "32f8acf8efda4681a32430ed441c03ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6f52ff104edd4429a4b240345b9862e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f1c0b05bdd2847f1b30e58fda0ecd781": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5dd81a67468b4f9cb96e77ba256407cb",
              "IPY_MODEL_a9c3e4ce33864d9e860c153eac150e8f",
              "IPY_MODEL_8f49df2faa6b43a8baa8b91ca8a963bb"
            ],
            "layout": "IPY_MODEL_64c79e440acc47dbbfc67343366cffee"
          }
        },
        "5dd81a67468b4f9cb96e77ba256407cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e0303e1c3ca44bf9931c3f87882e2c5f",
            "placeholder": "​",
            "style": "IPY_MODEL_84c82c664e53467ca7257c196b2e8713",
            "value": "100%"
          }
        },
        "a9c3e4ce33864d9e860c153eac150e8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "danger",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_39918ec44bb14e8386f77fabecd228ca",
            "max": 1193,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1c4db679f5034c28bda662cae98876ac",
            "value": 1192
          }
        },
        "8f49df2faa6b43a8baa8b91ca8a963bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_94d49b77582e4dd29108b4588518f2c2",
            "placeholder": "​",
            "style": "IPY_MODEL_1728273e4e574d8a88f65d6c471975da",
            "value": " 1192/1193 [00:00&lt;00:00,  9.08 examples/s]"
          }
        },
        "64c79e440acc47dbbfc67343366cffee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0303e1c3ca44bf9931c3f87882e2c5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "84c82c664e53467ca7257c196b2e8713": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "39918ec44bb14e8386f77fabecd228ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c4db679f5034c28bda662cae98876ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "94d49b77582e4dd29108b4588518f2c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1728273e4e574d8a88f65d6c471975da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "03f3dd0edb014d7aad618c14f0a23e45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e91d24def65d4f919260eae8241b348e",
              "IPY_MODEL_577c947153ba4f0f8f3184fe285b3f5e",
              "IPY_MODEL_1d42c71080fe429683033373b7dfae95"
            ],
            "layout": "IPY_MODEL_fdf49400a8a849c7b82f9ec7de41a0e6"
          }
        },
        "e91d24def65d4f919260eae8241b348e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_74023993ab184f34a2d48be2ee1bc0e5",
            "placeholder": "​",
            "style": "IPY_MODEL_e726d2f5a3324b6386c2097cc04fa511",
            "value": ""
          }
        },
        "577c947153ba4f0f8f3184fe285b3f5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5743bb425f8459481c92e857e23a544",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_75e2b4a26e214e70a0e7248f6b0f6109",
            "value": 1
          }
        },
        "1d42c71080fe429683033373b7dfae95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_55790e5c055e401c899f3d6cf563d4e8",
            "placeholder": "​",
            "style": "IPY_MODEL_1a8b4e79386046ecb1fc48f4a51ea877",
            "value": " 1631/0 [00:00&lt;00:00, 2720.73 examples/s]"
          }
        },
        "fdf49400a8a849c7b82f9ec7de41a0e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74023993ab184f34a2d48be2ee1bc0e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e726d2f5a3324b6386c2097cc04fa511": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b5743bb425f8459481c92e857e23a544": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "75e2b4a26e214e70a0e7248f6b0f6109": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "55790e5c055e401c899f3d6cf563d4e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1a8b4e79386046ecb1fc48f4a51ea877": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "45f7fae139a74e57b4bea932c4449973": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_41b9bd1d94fd4834b96088a2986268e1",
              "IPY_MODEL_0636d8f6befb4393a7fc9fa81fe1ba73",
              "IPY_MODEL_6ac043a386f04457ac6091c8947c61c7"
            ],
            "layout": "IPY_MODEL_b0b45bf3c7364ee1b5e8cfe90904f4d4"
          }
        },
        "41b9bd1d94fd4834b96088a2986268e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_217d0c12659b4f9d931f906ef0606330",
            "placeholder": "​",
            "style": "IPY_MODEL_5a90a44a2fed40038f434fc67464e71a",
            "value": "100%"
          }
        },
        "0636d8f6befb4393a7fc9fa81fe1ba73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "danger",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6add2cf9cbab43be82ff5c14ea038913",
            "max": 1803,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d9d4e6b00ac54a828aebb2f78620f212",
            "value": 1802
          }
        },
        "6ac043a386f04457ac6091c8947c61c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2bea25527b584d3f8c601e986d14db5e",
            "placeholder": "​",
            "style": "IPY_MODEL_495de7d68758488da7be12dc4404663d",
            "value": " 1802/1803 [00:00&lt;00:00,  6.50 examples/s]"
          }
        },
        "b0b45bf3c7364ee1b5e8cfe90904f4d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "217d0c12659b4f9d931f906ef0606330": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a90a44a2fed40038f434fc67464e71a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6add2cf9cbab43be82ff5c14ea038913": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9d4e6b00ac54a828aebb2f78620f212": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2bea25527b584d3f8c601e986d14db5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "495de7d68758488da7be12dc4404663d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}