Pretrain Model v4 - Combined Roberta Dataloader June 28 2020.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__How to train a language model__\tNotebook to Highlight all the steps to effectively train Transformer model on custom data\n",
    "https://github.com/huggingface/transformers/tree/master/notebooks\n",
    "\n",
    "https://github.com/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb\n",
    "\n",
    "__Language Modeling__\n",
    "https://github.com/huggingface/transformers/tree/master/examples/language-modeling\n",
    "https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_language_modeling.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1,2,3,4,5\"\n",
    "# os.environ[\"CUDA_LAUNCH_BLOCKING\"]=\"1\"\n",
    "\n",
    "from pathlib import Path\n",
    "import torch\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tokenizers import CharBPETokenizer, Tokenizer, ByteLevelBPETokenizer\n",
    "from tokenizers.processors import BertProcessing\n",
    "from tokenizers.normalizers import BertNormalizer\n",
    "# from tokenizers import SentencePieceBPETokenizer\n",
    "\n",
    "import random\n",
    "from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast , AutoTokenizer,RobertaTokenizerFast, RobertaTokenizer\n",
    "from filelock import FileLock\n",
    "import logging\n",
    "import time\n",
    "import tqdm\n",
    "import pickle\n",
    "from multiprocessing import Pool\n",
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cat123', 'dog123', 'mat123', 'pat123', 'rat123', 'bat123', 'bolo123']\n"
     ]
    }
   ],
   "source": [
    "array = [\"cat\", \"dog\", \"mat\", \"pat\", \"rat\", \"bat\" , \"bolo\"]\n",
    "def testadder(arrItem):\n",
    "    return arrItem+\"123\"\n",
    "\n",
    "with Pool(processes=3) as p:\n",
    "    examples = p.map(testadder, array)\n",
    "print(examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# device = torch.device(\"cuda:1\")\n",
    "# device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thu Jul  2 15:53:21 2020       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 450.36.06    Driver Version: 450.36.06    CUDA Version: 11.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla V100-SXM2...  Off  | 00000000:21:01.0 Off |                    0 |\n",
      "| N/A   30C    P0    55W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  Tesla V100-SXM2...  Off  | 00000000:21:02.0 Off |                    0 |\n",
      "| N/A   35C    P0    56W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  Tesla V100-SXM2...  Off  | 00000000:21:03.0 Off |                    0 |\n",
      "| N/A   34C    P0    57W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  Tesla V100-SXM2...  Off  | 00000000:21:04.0 Off |                    0 |\n",
      "| N/A   30C    P0    57W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   4  Tesla V100-SXM2...  Off  | 00000000:41:01.0 Off |                    0 |\n",
      "| N/A   35C    P0    56W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   5  Tesla V100-SXM2...  Off  | 00000000:41:02.0 Off |                    0 |\n",
      "| N/A   34C    P0    55W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   6  Tesla V100-SXM2...  Off  | 00000000:41:03.0 Off |                    0 |\n",
      "| N/A   38C    P0    45W / 300W |      3MiB / 16160MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   7  Tesla V100-SXM2...  Off  | 00000000:41:04.0 Off |                    0 |\n",
      "| N/A   37C    P0    53W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "# Check that PyTorch sees it\n",
    "print(torch.cuda.is_available())\n",
    "print(torch.cuda.device_count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../raw_data_extraction/thwiki-20200601-extracted/WikiAD_3.txt\n",
      "../raw_data_extraction/thwiki-20200601-extracted/WikiAE_3.txt\n",
      "../raw_data_extraction/thwiki-20200601-extracted/WikiAF_0.txt\n",
      "../raw_data_extraction/thwiki-20200601-extracted/WikiAF_2.txt\n",
      "../raw_data_extraction/thwiki-20200601-extracted/WikiAD_1.txt\n",
      "thwiki-20200601-extracted Amounts to a total of 566.79 MB\n",
      "../raw_data_extraction/classification_dataset/dailynews_0.txt\n",
      "../raw_data_extraction/classification_dataset/pptv36_0.txt\n",
      "../raw_data_extraction/classification_dataset/prbangkok_0.txt\n",
      "../raw_data_extraction/classification_dataset/siamrath_0.txt\n",
      "../raw_data_extraction/classification_dataset/springnews_0.txt\n",
      "classification_dataset Amounts to a total of 50.79 MB\n",
      "../raw_data_extraction/another_website/khaosod_16.txt\n",
      "../raw_data_extraction/another_website/pantip_470.txt\n",
      "../raw_data_extraction/another_website/pantip_415.txt\n",
      "../raw_data_extraction/another_website/naewna_2.txt\n",
      "../raw_data_extraction/another_website/brighttv_5.txt\n",
      "another_website Amounts to a total of 29552.82 MB\n",
      "../raw_data_extraction/data_lm/Pantipdata_train.csv_351.txt\n",
      "../raw_data_extraction/data_lm/Pantipdata_train.csv_81.txt\n",
      "../raw_data_extraction/data_lm/Pantipdata_train.csv_64.txt\n",
      "../raw_data_extraction/data_lm/Pantipdata_train.csv_118.txt\n",
      "../raw_data_extraction/data_lm/Pantipdata_train.csv_176.txt\n",
      "Senior Project Amounts to a total of 10942.78 MB\n",
      "../raw_data_extraction/social_listening/SocialListeningpantip_post_data.csv_5.txt\n",
      "../raw_data_extraction/social_listening/SocialListeningpantip_post_data.csv_1.txt\n",
      "../raw_data_extraction/social_listening/SocialListeningpantip_post_data.csv_3.txt\n",
      "../raw_data_extraction/social_listening/SocialListeningpantip_post_data.csv_2.txt\n",
      "../raw_data_extraction/social_listening/SocialListeningpantip_post_data.csv_0.txt\n",
      "GuruCrawler Amounts to a total of 171.21 MB\n",
      "\n",
      "I have a total of 1409 files!\n",
      "Amounts to a total of 41284.40 MB\n"
     ]
    }
   ],
   "source": [
    "DATA_PATH = Path(\"..\")\n",
    "\n",
    "# DATA_RAW_PATH = DATA_PATH/\"raw\"\n",
    "DATA_RAW_EXTRACTED_PATH = DATA_PATH/\"raw_data_extraction\"\n",
    "\n",
    "# Output is in bytes - helper from Pathlib Path https://stackoverflow.com/questions/2104080/how-can-i-check-file-size-in-python\n",
    "def getStat(prev_value, cur_value):\n",
    "    if isinstance(prev_value, int):\n",
    "        return prev_value + cur_value.stat().st_size\n",
    "    return prev_value.stat().st_size + cur_value.stat().st_size\n",
    "\n",
    "# 1. The data from thwiki\n",
    "THWIKI_FOLDER = Path(\"thwiki-20200601-extracted\")\n",
    "WIKI_FILES = list((DATA_RAW_EXTRACTED_PATH/THWIKI_FOLDER).glob(\"*.txt\"))\n",
    "list(map(print , WIKI_FILES[:5]))\n",
    "print(f\"thwiki-20200601-extracted Amounts to a total of {reduce(getStat, WIKI_FILES)/1e6:.2f} MB\")\n",
    "\n",
    "# 2. The classification data from jung and ninja\n",
    "CLASSIFICATION_JUNG_NINJA_FOLDER = Path(\"classification_dataset\")\n",
    "CLASSIFICATION_FILES = list((DATA_RAW_EXTRACTED_PATH/CLASSIFICATION_JUNG_NINJA_FOLDER).glob(\"*.txt\"))\n",
    "list(map(print , CLASSIFICATION_FILES[:5]))\n",
    "print(f\"classification_dataset Amounts to a total of {reduce(getStat, CLASSIFICATION_FILES)/1e6:.2f} MB\")\n",
    "\n",
    "# 3. The Data from p'Moo Crawlers\n",
    "ANOTHER_WEBSITE_MOO_FOLDER = Path(\"another_website\")\n",
    "ANOTHER_WEBSITE_FILES = list((DATA_RAW_EXTRACTED_PATH/ANOTHER_WEBSITE_MOO_FOLDER).glob(\"*.txt\"))\n",
    "list(map(print , ANOTHER_WEBSITE_FILES[:5]))\n",
    "print(f\"another_website Amounts to a total of {reduce(getStat, ANOTHER_WEBSITE_FILES)/1e6:.2f} MB\")\n",
    "\n",
    "# 4. Senior Project Files\n",
    "SENIOR_PROJ_FOLDER = Path(\"data_lm\")\n",
    "SENIOR_PROJ_FILES = list((DATA_RAW_EXTRACTED_PATH/SENIOR_PROJ_FOLDER).glob(\"*.txt\"))\n",
    "list(map(print , SENIOR_PROJ_FILES[:5]))\n",
    "print(f\"Senior Project Amounts to a total of {reduce(getStat, SENIOR_PROJ_FILES)/1e6:.2f} MB\")\n",
    "\n",
    "# 5. Guru Crawler Files\n",
    "GURU_CRAWLER_FOLDER = Path(\"social_listening\")\n",
    "GURU_CRAWLER_FILES = list((DATA_RAW_EXTRACTED_PATH/GURU_CRAWLER_FOLDER).glob(\"*.txt\"))\n",
    "list(map(print , GURU_CRAWLER_FILES[:5]))\n",
    "print(f\"GuruCrawler Amounts to a total of {reduce(getStat, GURU_CRAWLER_FILES)/1e6:.2f} MB\")\n",
    "\n",
    "ALL_FILES = WIKI_FILES + CLASSIFICATION_FILES + ANOTHER_WEBSITE_FILES + SENIOR_PROJ_FILES + GURU_CRAWLER_FILES\n",
    "print(f\"\\nI have a total of {len(ALL_FILES)} files!\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print(f\"Amounts to a total of {reduce(getStat, ALL_FILES)/1e6:.2f} MB\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making Electra Model\n",
    "\n",
    "However, currently pretraining Electra is still inside PR stage  \n",
    "- Issue : When will ELECTRA pretraining from scratch will be available? #3878 https://github.com/huggingface/transformers/issues/3878  \n",
    "- Issue : BERT and other models pretraining from scratch example #4425 https://github.com/huggingface/transformers/issues/4425\n",
    "- PR : Electra training from scratch #4656 https://github.com/huggingface/transformers/pull/4656\n",
    "\n",
    "Combined model\n",
    "- Combines ElectraForMaskedLM and ElectraForPreTraining with embedding sharing + custom masking/replaced token detection\n",
    "\n",
    "Commit before Merge: https://github.com/huggingface/transformers/pull/4656/commits/30b2dbbba6918ac6540b6a1758b7ee19f0ac969c#diff-8a95e2bfb7da25648b5d12ffa69fd7a3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from transformers import ElectraModel, ElectraConfig\n",
    "\n",
    "# # Initializing a ELECTRA electra-base-uncased style configuration\n",
    "# configuration = ElectraConfig()\n",
    "# configuration.vocab_size = 20000\n",
    "\n",
    "# # Initializing a model from the electra-base-uncased style configuration\n",
    "# model = ElectraModel(configuration)\n",
    "\n",
    "# # Accessing the model configuration\n",
    "# configuration = model.config\n",
    "# configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.num_parameters()\n",
    "# # => 12 million parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.get_input_embeddings()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trying out Roberta per Notebook \n",
    "\n",
    "From __HuggingFace Notebooks__ https://huggingface.co/transformers/notebooks.html: \n",
    "\n",
    "How to train a language model\tHighlight all the steps to effectively train Transformer model on custom data\n",
    "- Colab (ipynb) version : https://github.com/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb\n",
    "- MD version: https://github.com/huggingface/blog/blob/master/how-to-train.md\n",
    "\n",
    "Pretrain Longformer\tHow to build a \"long\" version of existing pretrained models\tIz Beltagy  \n",
    "https://github.com/allenai/longformer/blob/master/scripts/convert_model_to_long.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RobertaConfig {\n",
       "  \"attention_probs_dropout_prob\": 0.1,\n",
       "  \"bos_token_id\": 0,\n",
       "  \"eos_token_id\": 2,\n",
       "  \"gradient_checkpointing\": false,\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"layer_norm_eps\": 1e-12,\n",
       "  \"max_position_embeddings\": 512,\n",
       "  \"model_type\": \"roberta\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_hidden_layers\": 12,\n",
       "  \"pad_token_id\": 1,\n",
       "  \"type_vocab_size\": 1,\n",
       "  \"vocab_size\": 20000\n",
       "}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import RobertaConfig\n",
    "from transformers import RobertaForMaskedLM\n",
    "\n",
    "configuration = RobertaConfig(\n",
    "    vocab_size=20000,\n",
    "    max_position_embeddings=512, # 512 + 2 more special tokens\n",
    "    num_attention_heads=12,\n",
    "    num_hidden_layers=12,\n",
    "    type_vocab_size=1,\n",
    ")\n",
    "# configuration.vocab_size = 20000\n",
    "\n",
    "model = RobertaForMaskedLM(config=configuration)\n",
    "\n",
    "# Accessing the model configuration\n",
    "model.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102012704"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.num_parameters()\n",
    "# => 102 million parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RobertaForMaskedLM(\n",
       "  (roberta): RobertaModel(\n",
       "    (embeddings): RobertaEmbeddings(\n",
       "      (word_embeddings): Embedding(20000, 768, padding_idx=1)\n",
       "      (position_embeddings): Embedding(512, 768, padding_idx=1)\n",
       "      (token_type_embeddings): Embedding(1, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (lm_head): RobertaLMHead(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "    (decoder): Linear(in_features=768, out_features=20000, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initializing Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tokenizers import Tokenizer\n",
    "# tokenizer = Tokenizer.from_file(\"./thwiki-sentencepiecebpe.tokenizer.json\")\n",
    "# encoded =  tokenizer.encode(u\"สวัสดีครับ ผมชื่อไนท์ ตอนนี้ก็เป็นเวลาที่ผมต้องไปโรงเรียนแล้ว  นี่คือการเว้นวรรคสองทีครับ  จะได้ออกเป็นสอง Spaces\")\n",
    "# print(encoded.ids)\n",
    "# print(encoded.tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizer.enable_truncation(max_length=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoded =  tokenizer.encode(u\"สวัสดีครับ ผมชื่อไนท์ ตอนนี้ก็เป็นเวลาที่ผมต้องไปโรงเรียนแล้ว  นี่คือการเว้นวรรคสองทีครับ  จะได้ออกเป็นสอง SpacesWhat is great is that our tokenizer is optimized for Esperanto. Compared to a generic tokenizer trained for English, more native words are represented by a single, unsplit token. Diacritics, i.e. accented characters used in Esperanto – ĉ, ĝ, ĥ, ĵ, ŝ, and ŭ – are encoded natively. We also represent sequences in a more efficient manner. Here on this corpus, the average length of encoded sequences is ~30% smaller as when using the pretrained GPT-2 tokenizer.\")\n",
    "# print(\"This will not be over 128: \", len(encoded.ids), encoded.tokens)\n",
    "# print(encoded.overflowing[0].tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "wrap tokenizers inside a PreTrainedTokenizerFast from transformers \n",
    "\n",
    "https://github.com/huggingface/tokenizers/issues/259"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class SentencePieceBPETokenizerFast(PreTrainedTokenizerFast):\n",
    "#     def __init__(\n",
    "#         self,\n",
    "#         vocab_file,\n",
    "#         merges_file,\n",
    "#         bos_token=\"<s>\",\n",
    "#         eos_token=\"</s>\",\n",
    "#         sep_token=\"</s>\",\n",
    "#         cls_token=\"<s>\",\n",
    "#         unk_token=\"<unk>\",\n",
    "#         pad_token=\"<pad>\",\n",
    "#         mask_token=\"<mask>\",\n",
    "#         **kwargs\n",
    "#     ):\n",
    "#         super().__init__(\n",
    "#             SentencePieceBPETokenizer(\n",
    "#                 vocab_file=vocab_file,\n",
    "#                 merges_file=merges_file,\n",
    "#             ),\n",
    "#              bos_token=bos_token,\n",
    "#             eos_token=eos_token,\n",
    "#             unk_token=unk_token,\n",
    "#             sep_token=sep_token,\n",
    "#             cls_token=cls_token,\n",
    "#             pad_token=pad_token,\n",
    "#             mask_token=mask_token,\n",
    "#             **kwargs,\n",
    "#         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import json\n",
    "\n",
    "# # with open(\"./thwiki-sentencepiecebpe.tokenizer.json\", 'r' ) as json_data:\n",
    "# with open(\"./thwiki-charbpe-30522.tokenizer.json\", 'r' ) as json_data:\n",
    "#      data = json.load(json_data)\n",
    "# vocab = data['model']['vocab']\n",
    "# merges = data['model']['merges']\n",
    "\n",
    "\n",
    "# with open('vocab.json', 'w', encoding='utf-8') as json_file:\n",
    "#     json.dump(vocab, json_file, ensure_ascii=False)\n",
    "# with open('merges.txt', 'w', encoding='utf-8') as f:\n",
    "#     for merge_string in merges:\n",
    "#         f.write(f'{merge_string}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pretrain_tokenizer = SentencePieceBPETokenizerFast(vocab_file='vocab.json',merges_file ='merges.txt' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast\n",
    "# from tokenizers import Tokenizer\n",
    "# from tokenizers.implementations import BaseTokenizer\n",
    "\n",
    "# tokenizer = Tokenizer.from_file(\"./thwiki-sentencepiecebpe.tokenizer.json\")\n",
    "# base_tokenizer = BaseTokenizer(tokenizer) # Wrapper!! to PretrainTokenizerFast Tokenizer should be an instance of a Tokenizer provided by HuggingFace tokenizers library.\n",
    "# base_tokenizer = SentencePieceBPETokenizer()\n",
    "# pretrain_tokenizer = PreTrainedTokenizerFast(tokenizer=base_tokenizer)\n",
    "# pretrain_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast\n",
    "# from tokenizers import Tokenizer, CharBPETokenizer\n",
    "# from tokenizers.implementations import BaseTokenizer\n",
    "\n",
    "# # tokenizer = Tokenizer.from_file(\"./thwiki-charbpe-30522.tokenizer.json\")\n",
    "# # base_tokenizer = BaseTokenizer(tokenizer) # Wrapper!! to PretrainTokenizerFast Tokenizer should be an instance of a Tokenizer provided by HuggingFace tokenizers library.\n",
    "# base_tokenizer = CharBPETokenizer(vocab_file='vocab.json',merges_file ='merges.txt')\n",
    "# pretrain_tokenizer = PreTrainedTokenizerFast(tokenizer=base_tokenizer)\n",
    "# pretrain_tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CANNOT USE FAST BECAUSE multiprocessing Pool doesn't support Rust!!\n",
    "Using %%time, there is almost x7-x10 difference in tokenization but we cannot not allow multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from transformers import RobertaTokenizerFast\n",
    "\n",
    "# tokenizer = RobertaTokenizerFast.from_pretrained(\"./all-data-bytebpe-20000\", max_len=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'RobertaTokenizer'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# from transformers import RobertaTokenizer\n",
    "\n",
    "tokenizer = RobertaTokenizer.from_pretrained(\"./all-data-bytebpe-20000\", max_len=512)\n",
    "tokenizer.__class__.__name__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "2\n",
      "['à¸ī', 'à¸±', 'à¸Ļà¹Ģà¸Ħà¸¢', 'à¹Ģà¸ģ', 'à¸·', 'à¸Ńà¸ļ', 'à¸ŀà¸¥à¸²à¸Ķ', 'à¸ª', 'à¸´à¹Ī', 'à¸ĩà¸Ĺ', 'à¸µà¹Ī', 'à¸Ķ', 'à¸µ', 'à¸Ĺ', 'à¸µà¹Ī', 'à¸ª', 'à¸¸', 'à¸Ķà¹ĥà¸Ļà¸Ĭ', 'à¸µ', 'à¸§', 'à¸´', 'à¸ķ', 'Ġà¸«à¸²à¸ģ', 'à¹ĥà¸Ļà¸§', 'à¸±', 'à¸Ļà¸Ĺ', 'à¸µà¹Ī', 'à¸ī', 'à¸±', 'à¸Ļà¸¥', 'à¹ī', 'à¸¡à¸Ńà¸¢', 'à¸¹à¹Ī', 'Ġà¹Ħà¸¡', 'à¹Ī', 'à¸¡', 'à¸µ', 'à¸«à¸Ļ', 'à¸¶à¹Ī', 'à¸ĩà¹ĥà¸Ī', 'à¸Ĥà¸Ńà¸ĩà¹Ģà¸ĺà¸Ń', 'Ġà¸Ŀ', 'à¸±', 'à¸Ļà¸Ħ', 'à¸ĩà¸Īà¸ļ', 'Ġà¸«à¸¥à¸²à¸¢', 'à¸ª', 'à¸´à¹Ī', 'à¸ĩà¸Ĺ', 'à¸µà¹Ī', 'à¸Ķ', 'à¸µ', 'à¸Ħ', 'à¸ĩà¸«à¸¡à¸Ķ', 'à¸Ĺà¸²à¸ĩ', 'à¹Ħà¸Ķ', 'à¹ī', 'à¹Ģà¸Īà¸Ń', 'Ġà¸«à¸Ļ', 'à¸¶à¹Ī', 'à¸ĩà¸ģ', 'à¸', '³', 'à¸¥', 'à¸±', 'à¸ĩà¹ĥà¸Īà¸Ĺ', 'à¸µà¹Ī', 'à¸¢', 'à¸´à¹Ī', 'à¸ĩà¹ĥà¸«à¸į', 'à¹Ī', 'Ġà¹Ħà¸¡', 'à¹Ī', 'à¸¥', 'à¸·', 'à¸¡à¹Ħà¸Ķ', 'à¹ī', 'à¹Ģà¸¥à¸¢', '...', 'Ġà¹ĥà¸Ħà¸£à¹Ģà¸Ħà¸¢à¸¡', 'à¸µ', 'à¹ģà¸Łà¸Ļà¸Ĺ', 'à¸µà¹Ī', 'à¸ģ', 'à¸´', 'à¸Ļà¸Ńà¸²à¸«à¸²à¸£', 'à¹Ħà¸¡', 'à¹Ī', 'à¸ĸ', 'à¸¹', 'à¸ģà¸Ľ', 'à¸²à¸ģà¸ģ', 'à¸±', 'à¸Ļà¹ģà¸¥', 'à¹ī', 'à¸§à¸£', 'à¸¹à¹ī', 'à¸ª', 'à¸¶', 'à¸ģà¹Ģà¸ª', 'à¸µ', 'à¸¢', 'à¸Ħà¸§à¸²à¸¡à¸ª', 'à¸¸', 'à¸Ĥ', 'à¹Ħà¸Ľà¸Ńà¸¢', 'à¹Ī', 'à¸²à¸ĩà¸Ļ', 'à¸¶', 'à¸ĩà¸ļ', 'à¹ī', 'à¸²à¸ĩà¸¡', 'à¸±à¹ī', 'à¸¢à¸Ħà¸£', 'à¸±', 'à¸ļ', 'Ġ', 'Ġà¸ģ', 'à¹Ī', 'à¸Ńà¸Ļà¸Ń', 'à¸·à¹Ī', 'à¸Ļà¸ľà¸¡', 'à¸ķ', 'à¹ī', 'à¸Ńà¸ĩà¸ļà¸Ńà¸ģà¸ģ', 'à¹Ī', 'à¸Ńà¸Ļà¹Ģà¸¥à¸¢à¸§', 'à¹Ī', 'à¸²à¸Ħ', 'à¸Ļà¹Ģà¸£', 'à¸²à¸Īà¸°à¹Ģà¸¥', 'à¸·', 'à¸Ńà¸ģà¸ģ', 'à¸´', 'à¸Ļà¸Ńà¸²à¸«à¸²à¸£', 'à¹ģà¸ļà¸ļà¹Ħà¸«à¸Ļ', 'à¸Ĭà¸Ńà¸ļ', 'à¹ģà¸ļà¸ļà¹Ħà¸«à¸Ļ', 'à¹Ģà¸Ľ', 'à¹ĩ', 'à¸Ļà¹Ģà¸£', 'à¸·à¹Ī', 'à¸Ńà¸ĩà¸Ĥà¸Ńà¸ĩ', 'à¸Ħà¸§à¸²à¸¡', 'à¸Ĭà¸Ńà¸ļà¸ª', 'à¹Ī', 'à¸§à¸Ļà¸ķ', 'à¸±', 'à¸§à¸Ļà¸°à¸Ħà¸£', 'à¸±', 'à¸ļà¸Ĺ', 'à¸¸', 'à¸ģà¸Ħà¸Ļà¸¡', 'à¸µ', 'à¸ª', 'à¸´', 'à¸Ĺà¸ĺ', 'à¸´', 'à¹ĥà¸Ļà¸ģà¸²à¸£à¹Ģà¸¥', 'à¸·', 'à¸Ńà¸ģ', 'à¸Ĥà¸Ńà¸ĩà¸Ĺ', 'à¸µà¹Ī', 'à¸Ĭà¸Ńà¸ļ', 'à¹ģà¸¥à¸°à¹Ħà¸¡', 'à¹Ī', 'à¸Ĭà¸Ńà¸ļà¸Ńà¸¢', 'à¸¹à¹Ī', 'à¹ģà¸¥', 'à¹ī', 'à¸§', 'Ġà¹ģà¸ķ', 'à¹Ī', 'à¸ľà¸¡à¸£', 'à¸¹à¹ī', 'à¸ª', 'à¸¶', 'à¸ģà¸§', 'à¹Ī', 'à¸²à¸ķà¸Ńà¸Ļà¸Ļ', 'à¸µà¹ī', 'à¸ľà¸¡à¸ģ', 'à¸', '³', 'à¸¥', 'à¸±', 'à¸ĩ', 'à¸Ľà¸£à¸°à¸ªà¸ļà¸Ľ', 'à¸±', 'à¸įà¸«à¸²à¸Ĺ', 'à¸µà¹Ī', 'à¸Ķ', 'à¸¹', 'à¹Ģà¸«à¸¡', 'à¸·', 'à¸Ńà¸Ļà¸Īà¸°', 'à¹Ģà¸¥', 'à¹ĩ', 'à¸ģà¹ģà¸ķ', 'à¹Ī', 'à¸ģà¸¥à¸²à¸¢à¹Ģà¸Ľ', 'à¹ĩ', 'à¸Ļà¸§', 'à¹Ī', 'à¸²à¸¡', 'à¸±', 'à¸Ļà¸Ħ', 'à¹Ī', 'à¸Ńà¸Ļà¸Ĥ', 'à¹ī', 'à¸²à¸ĩà¹ĥà¸«à¸į', 'à¹Ī', 'Ġà¸ľà¸¡', 'à¸Ħà¸ļà¸ģ', 'à¸±', 'à¸ļà¹ģà¸Łà¸Ļà¸¡à¸²', '6', 'à¸Ľ', 'à¸µ', 'à¹ģà¸¥', 'à¹ī', 'à¸§à¸Ħà¸£', 'à¸±', 'à¸ļ', 'Ġà¸ľà¸¡à¹Ģà¸Ľ', 'à¹ĩ', 'à¸Ļà¸Ħà¸Ļ', 'à¸Ĭà¸Ńà¸ļà¸ģ', 'à¸´', 'à¸Ļà¸Ńà¸²à¸«à¸²à¸£', 'à¸į', 'à¸µà¹Ī', 'à¸Ľ', 'à¸¸à¹Ī', 'à¸Ļà¹ģà¸¥à¸°', 'à¸Ľà¸¥', 'à¸²à¸Ķ', 'à¸´', 'à¸ļà¹ģà¸ķ', 'à¹Ī', 'à¹ģà¸Łà¸Ļ', 'à¸ľà¸¡à¹Ħà¸¡', 'à¹Ī', 'à¸ģ', 'à¸´', 'à¸Ļà¸Ľà¸¥', 'à¸²à¸Ķ', 'à¸´', 'à¸ļà¹Ģà¸¥à¸¢', 'Ġà¸ľà¸¡à¸Ńà¸¢à¸²à¸ģ', 'à¸ģ', 'à¸´', 'à¸Ļà¸ļ', 'à¸¸', 'à¸Łà¹Ģà¸Ł', 'à¹Ī', 'à¹Ģà¸Ļ', 'à¸·à¹ī', 'à¸Ńà¹ģà¸ķ', 'à¹Ī', 'à¹ģà¸Łà¸Ļ', 'à¸ľà¸¡à¸ģ', 'à¹ĩ', 'à¹Ħà¸¡', 'à¹Ī', 'à¸ģ', 'à¸´', 'à¸Ļà¹Ģà¸Ļ', 'à¸·à¹ī', 'à¸Ń', 'Ġà¹Ģà¸£à¸²à¹Ģà¸¥à¸¢à¹Ħà¸¡', 'à¹Ī', 'à¹Ħà¸Ķ', 'à¹ī', 'à¹Ģà¸Ĥ', 'à¹ī', 'à¸²à¸Ĺ', 'à¸²à¸Ļà¸£', 'à¹ī', 'à¸²à¸Ļà¸ļ', 'à¸¸', 'à¸Łà¹Ģà¸Ł', 'à¹Ī', 'à¹Ģà¸Ļ', 'à¸·à¹ī', 'à¸Ńà¹ģà¸¥à¸°', 'à¸ļ', 'à¸¸', 'à¸Łà¹Ģà¸Ł', 'à¹Ī', 'à¸Ńà¸²à¸«à¸²à¸£', 'à¸į', 'à¸µà¹Ī', 'à¸Ľ', 'à¸¸à¹Ī', 'à¸Ļà¸ģ', 'à¸±', 'à¸Ļà¹Ģà¸ŀà¸£à¸²à¸°', 'à¸£', 'à¸¹à¹ī', 'à¸ª', 'à¸¶', 'à¸ģà¸¥', 'à¸±', 'à¸§', 'à¹ģà¸Łà¸Ļà¸ľà¸¡', 'à¸Ĺà¸²à¸Ļ', 'à¹Ħà¸¡', 'à¹Ī', 'à¸Ħ', 'à¸¸à¹ī', 'à¸¡', 'Ġà¹ģà¸¥à¸°à¹Ģà¸£', 'à¸·à¹Ī', 'à¸Ńà¸ĩà¹ĥà¸«à¸į', 'à¹Ī', 'à¹Ģà¸¥à¸¢à¸Ħ', 'à¸·', 'à¸Ńà¸ľà¸¡à¹Ģà¸Ľ', 'à¹ĩ', 'à¸Ļà¸Ħà¸Ļà¸Ĭà¸Ńà¸ļ', 'à¸Ĺà¸²à¸Ļà¸Ńà¸²à¸«à¸²à¸£', 'à¸£à¸ª', 'à¸Ī', 'à¸±', 'à¸Ķà¹ģà¸¥à¸°', 'à¸£à¸ª', 'à¹Ģà¸ľ', 'à¹ĩ', 'à¸Ķà¸¡à¸²à¸ģ', 'Ġà¹ģà¸ķ', 'à¹Ī', 'à¹ģà¸Łà¸Ļà¸ľà¸¡', 'à¸Ĺà¸²à¸Ļ', 'à¹Ģà¸ľ', 'à¹ĩ', 'à¸Ķà¹Ħà¸¡', 'à¹Ī', 'à¹Ħà¸Ķ', 'à¹ī', 'à¹Ģà¸¥à¸¢', 'à¹Ģà¸§à¸¥à¸²', 'à¹Ģà¸£à¸²', 'à¹Ħà¸Ľà¸ģ', 'à¸´', 'à¸Ļà¸ª', 'à¹ī', 'à¸¡à¸ķ', 'à¸', '³', 'à¸ģ', 'à¸±', 'à¸Ļà¸ģ', 'à¹ĩ', 'à¸Īà¸°à¸ª', 'à¸±à¹Ī', 'à¸ĩ', 'Ġà¸ª', 'à¹ī', 'à¸¡à¸ķ', 'à¸', '³', 'à¹Ħà¸¡', 'à¹Ī', 'à¹ĥà¸ª', 'à¹Ī', 'à¸ŀà¸£', 'à¸´', 'à¸ģ', 'Ġà¸ķ', 'à¹ī', 'à¸¡', 'à¹ģà¸ĭ', 'à¹Ī', 'à¸ļà¹Ħà¸¡', 'à¹Ī', 'à¹ĥà¸ª', 'à¹Ī', 'à¸ŀà¸£', 'à¸´', 'à¸ģ', 'Ġà¸¥', 'à¸²à¸ļ', 'à¹Ħà¸¡', 'à¹Ī', 'à¹ĥà¸ª', 'à¹Ī', 'à¸ŀà¸£', 'à¸´', 'à¸ģ', 'Ġà¸£', 'à¹ī', 'à¸²à¸Ļà¸ģ', 'à¸±', 'à¸ļà¸Ĥ', 'à¹ī', 'à¸²à¸§à¸Ń', 'à¸·à¹Ī', 'à¸Ļà¹Ĩà¸ģ', 'à¹ĩ', 'à¹Ģà¸Ĭ', 'à¹Ī', 'à¸Ļà¸ģ', 'à¸±', 'à¸Ļà¹ģà¸Łà¸Ļ', 'à¸ľà¸¡', 'à¸Īà¸°à¹Ħà¸¡', 'à¹Ī', 'à¸Ĭà¸Ńà¸ļà¸ģ', 'à¸´', 'à¸Ļà¸ľ', 'à¸±', 'à¸ģà¹Ħà¸¡', 'à¹Ī', 'à¸Ħ', 'à¹Ī', 'à¸Ńà¸¢à¸ª', 'à¸±à¹Ī', 'à¸ĩà¸ģ', 'à¸±', 'à¸ļà¸Ĥ', 'à¹ī', 'à¸²à¸§à¸Ĺ', 'à¸µà¹Ī', 'à¹Ģà¸Ľ', 'à¹ĩ', 'à¸Ļà¸ľ', 'à¸±', 'à¸ģà¹ģà¸¥', 'à¹ī', 'à¸§à¸ľà¸¡', 'à¸Ĭà¸Ńà¸ļà¸ľ', 'à¸±', 'à¸ģà¸ļ', 'à¸¸à¹ī', 'à¸ĩà¸Ĺ', 'à¸Ńà¸Ķ', 'à¸ģà¸£à¸Ńà¸ļ', 'Ġà¹Ģà¸«', 'à¹ĩ', 'à¸Ķà¸«', 'à¸Ńà¸¡', 'à¸ªà¸Ķ', 'à¸Ĺà¸Ńà¸Ķ', 'à¸¡à¸²à¸ģ', 'Ġà¹ģà¸ķ', 'à¹Ī', 'à¸ģ', 'à¹ĩ', 'à¹Ħà¸¡', 'à¹Ī', 'à¹Ħà¸Ķ', 'à¹ī', 'à¸ª', 'à¸±à¹Ī', 'à¸ĩ', 'à¹Ģà¸ŀà¸£à¸²à¸°à¸§', 'à¹Ī', 'à¸²à¹Ģà¸ĺ', 'à¸Ńà¹Ħà¸¡', 'à¹Ī', 'à¸ģ', 'à¸´', 'à¸Ļà¸ĸ', 'à¸¶', 'à¸ĩà¹Ģà¸Ħ', 'à¹ī', 'à¸²à¸Īà¸°', 'à¸ļà¸Ńà¸ģà¹ĥà¸«', 'à¹ī', 'à¸ª', 'à¸±à¹Ī', 'à¸ĩà¹Ģà¸¥à¸¢', 'à¹Ĩà¸ģ', 'à¹ĩ', 'à¹Ģà¸ĸà¸Ńà¸°', 'à¹ģà¸ķ', 'à¹Ī', 'à¸ľà¸¡à¸ģ', 'à¹ĩ', 'à¸¢', 'à¸±', 'à¸ĩà¹Ģà¸ģ', 'à¸£', 'à¸ĩà¹ĥà¸Ī', 'à¹Ģà¸ĺà¸Ńà¸Ńà¸¢', 'à¸¹à¹Ī', 'à¸Ķ', 'à¸µ', 'à¸Ń', 'à¹Ī', 'à¸°à¸Ħà¸£', 'à¸±', 'à¸ļ', 'Ġà¸ľà¸¡à¸£', 'à¸¹à¹ī', 'à¸ª', 'à¸¶', 'à¸ģà¸ģ', 'à¸´', 'à¸Ļà¸Ńà¸²à¸«à¸²à¸£', 'à¹Ħà¸¡', 'à¹Ī', 'à¸¡', 'à¸µ', 'à¸Ħà¸§à¸²à¸¡à¸ª', 'à¸¸', 'à¸Ĥ', 'à¹Ģà¸¥à¸¢', 'à¸Ĭ', 'à¸µ', 'à¸§', 'à¸´', 'à¸ķà¸ľà¸¡', 'à¸Ĥà¸²à¸Ķ', 'à¸£à¸ª', 'à¹Ģà¸ľ', 'à¹ĩ', 'à¸Ķà¹Ħà¸Ľ', 'à¹Ģà¸«à¸¡', 'à¸·', 'à¸Ńà¸Ļà¸Īà¸°', 'à¸Ĥà¸²à¸Ķ', 'à¹ĥà¸Ī', 'à¹Ģà¸«à¸¡', 'à¸·', 'à¸Ńà¸Ļà¸¡', 'à¸±', 'à¸Ļà¸Ĺ', 'à¸', '³', 'à¹ĥà¸«', 'à¹ī', 'à¸Ĥà¸²à¸Ķ', 'à¸Ħà¸§à¸²à¸¡à¸ª', 'à¸¸', 'à¸Ĥ', 'à¹Ħà¸Ľà¸Ńà¸¢', 'à¹Ī', 'à¸²à¸ĩà¸Ļ', 'à¸¶', 'à¸ĩà¹Ģà¸¥à¸¢', 'à¸Ń', 'à¹Ī', 'à¸°à¸Ħà¸£', 'à¸±', 'à¸ļ', 'Ġà¸¢', 'à¸´à¹Ī', 'à¸ĩà¸ĸ', 'à¹ī', 'à¸²à¹Ģà¸£à¸²', 'à¹ģà¸ķ', 'à¹Ī', 'à¸ĩà¸ĩà¸²à¸Ļà¸ģ', 'à¸±', 'à¸Ļà¹ģà¸¥', 'à¹ī', 'à¸§à¸ľà¸¡à¸ģ', 'à¹ĩ', 'à¸Ńà¸²à¸Īà¸Īà¸°à¸ķ', 'à¹ī', 'à¸Ńà¸ĩà¸¡', 'à¸µ', 'à¸Ľ', 'à¸±', 'à¸įà¸«à¸²à¹Ģà¸£', 'à¸·à¹Ī', 'à¸Ńà¸ĩà¸Ļ', 'à¸µà¹ī', 'à¸¡à¸²à¸ģà¸Ĥ', 'à¸¶à¹ī', 'à¸Ļ', 'Ġà¸ŀà¸Ń', 'à¸ľà¸¡à¹Ģà¸«', 'à¹ĩ', 'à¸Ļà¸Ħ', 'à¸¹à¹Ī', 'à¸Ĺ', 'à¸µà¹Ī', 'à¸Ĭà¸Ńà¸ļ', 'à¸Ĺà¸²à¸Ļà¸Ńà¸²à¸«à¸²à¸£', 'à¹Ģà¸«à¸¡', 'à¸·', 'à¸Ńà¸Ļà¹Ĩà¸ģ', 'à¸±', 'à¸Ļà¹Ģà¸«', 'à¹ĩ', 'à¸Ļà¹Ģà¸Ħ', 'à¹ī', 'à¸²à¸ģ', 'à¸´', 'à¸Ļà¸Ńà¸²à¸«à¸²à¸£', 'à¸ģ', 'à¸±', 'à¸Ļà¸Ńà¸¢', 'à¹Ī', 'à¸²à¸ĩà¸¡', 'à¸µ', 'à¸Ħà¸§à¸²à¸¡à¸ª', 'à¸¸', 'à¸Ĥ', 'à¹ģà¸¥', 'à¹ī', 'à¸§à¸ľà¸¡', 'à¸£', 'à¸¹à¹ī', 'à¸ª', 'à¸¶', 'à¸ģà¸Ń', 'à¸´', 'à¸Īà¸ī', 'à¸²à¸¡à¸²à¸ģà¹Ĩ', 'à¹Ģà¸¥à¸¢', 'Ġà¸¡', 'à¸µ', 'à¹ĥà¸Ħà¸£', 'à¹Ģà¸Ħà¸¢à¸¡', 'à¸µ', 'à¸Ľ', 'à¸±', 'à¸įà¸«à¸²', 'à¹ģà¸ļà¸ļ', 'à¸ľà¸¡à¸¡', 'à¸±à¹ī', 'à¸¢à¸Ħà¸£', 'à¸±', 'à¸ļà¹ģà¸¥', 'à¹ī', 'à¸§à¸Īà¸°', 'à¹ģà¸ģ', 'à¹ī', 'à¸Ľ', 'à¸±', 'à¸įà¸«à¸²à¸Ļ', 'à¸µà¹ī', 'à¸¢', 'à¸±', 'à¸ĩà¹Ħà¸ĩà¸Ķ', 'à¸µ', 'à¸Ħà¸£', 'à¸±', 'à¸ļ'] with length 643\n",
      "[560, 275, 5373, 363, 296, 349, 1647, 294, 406, 455, 302, 281, 278, 288, 302, 294, 305, 9422, 278, 279, 289, 290, 1266, 1630, 275, 368, 302, 560, 275, 843, 271, 3299, 370, 449, 266, 276, 278, 343, 440, 990, 8445, 1302, 275, 501, 12132, 2562, 294, 406, 455, 302, 281, 278, 283, 1217, 527, 332, 271, 912, 782, 440, 500, 261, 116, 284, 275, 5425, 302, 280, 406, 3013, 266, 449, 266, 284, 296, 1978, 271, 380, 642, 16378, 278, 7942, 302, 274, 289, 3645, 320, 266, 326, 304, 1439, 6178, 275, 1018, 271, 697, 345, 294, 330, 2377, 278, 280, 883, 305, 306, 3720, 266, 949, 330, 852, 271, 1373, 339, 1544, 275, 282, 225, 361, 266, 2353, 327, 2193, 290, 271, 9443, 266, 13457, 266, 354, 780, 5164, 296, 3251, 289, 3645, 4311, 596, 4311, 334, 301, 780, 327, 1426, 375, 10907, 266, 1088, 275, 5717, 275, 635, 305, 5843, 278, 294, 289, 637, 289, 6988, 296, 341, 1921, 302, 596, 3420, 266, 13802, 370, 324, 271, 279, 397, 266, 3525, 345, 294, 330, 438, 266, 3115, 340, 1179, 261, 116, 284, 275, 277, 10287, 275, 3676, 302, 281, 304, 558, 296, 2028, 338, 301, 2880, 266, 2119, 301, 531, 266, 319, 275, 501, 266, 1321, 271, 10271, 266, 530, 2631, 275, 15047, 26, 297, 278, 324, 271, 1392, 275, 282, 3901, 301, 705, 4939, 289, 3645, 356, 302, 297, 450, 1216, 553, 411, 289, 2571, 266, 746, 1925, 266, 274, 289, 4679, 411, 289, 4355, 6407, 274, 289, 718, 305, 6747, 266, 457, 426, 3358, 266, 746, 1179, 301, 320, 266, 274, 289, 1955, 426, 267, 14864, 266, 332, 271, 357, 271, 359, 2602, 271, 2926, 305, 6747, 266, 457, 426, 2906, 282, 305, 6747, 266, 1143, 356, 302, 297, 450, 456, 275, 1824, 273, 345, 294, 330, 381, 275, 279, 9399, 1591, 320, 266, 283, 619, 276, 5392, 327, 5456, 266, 889, 296, 12955, 301, 7998, 14431, 2094, 295, 275, 3242, 2094, 906, 301, 1819, 397, 266, 9399, 1591, 906, 301, 1754, 266, 332, 271, 380, 885, 459, 1385, 289, 478, 271, 814, 261, 116, 274, 275, 456, 301, 2531, 417, 277, 364, 271, 814, 261, 116, 320, 266, 656, 266, 539, 289, 274, 382, 271, 276, 2023, 266, 1397, 266, 656, 266, 539, 289, 274, 523, 480, 320, 266, 656, 266, 539, 289, 274, 409, 271, 1282, 275, 1065, 271, 3158, 327, 13024, 301, 461, 266, 456, 275, 3402, 390, 1224, 266, 4939, 289, 838, 275, 1531, 266, 283, 266, 2647, 417, 500, 275, 1065, 271, 1855, 302, 334, 301, 838, 275, 1561, 271, 2371, 6514, 275, 1564, 619, 455, 412, 2031, 651, 301, 1869, 384, 566, 8029, 396, 397, 266, 274, 301, 320, 266, 332, 271, 294, 417, 277, 4084, 266, 4760, 769, 266, 274, 289, 951, 330, 3194, 271, 465, 3148, 271, 294, 417, 2114, 2573, 301, 2005, 420, 266, 1179, 301, 280, 275, 1046, 273, 990, 16224, 370, 281, 278, 267, 266, 1232, 275, 282, 4112, 345, 294, 330, 521, 289, 3645, 320, 266, 276, 278, 883, 305, 306, 380, 307, 278, 279, 289, 18332, 1867, 2094, 906, 301, 1609, 558, 296, 2028, 1867, 400, 558, 296, 1556, 275, 368, 261, 116, 336, 271, 1867, 883, 305, 306, 3720, 266, 949, 330, 2114, 267, 266, 1232, 275, 282, 538, 406, 1253, 271, 1361, 420, 266, 4993, 275, 1018, 271, 5804, 301, 10164, 271, 813, 278, 297, 275, 4509, 327, 711, 340, 1902, 458, 264, 1202, 8652, 301, 501, 370, 288, 302, 596, 14431, 558, 296, 18548, 275, 4111, 301, 2435, 271, 310, 289, 3645, 274, 275, 730, 266, 1373, 278, 883, 305, 306, 324, 271, 2371, 273, 345, 294, 330, 1330, 289, 2460, 9131, 380, 371, 278, 512, 2064, 278, 297, 275, 1750, 454, 2947, 339, 1544, 275, 2587, 271, 2129, 568, 271, 297, 275, 4915, 340, 280, 275, 1831, 278, 333, 275, 282] with length 643\n",
      "[0, 560, 275, 5373, 363, 296, 349, 1647, 294, 406, 455, 302, 281, 278, 288, 302, 294, 305, 9422, 278, 279, 289, 290, 1266, 1630, 275, 368, 302, 560, 275, 843, 271, 3299, 370, 449, 266, 276, 278, 343, 440, 990, 8445, 1302, 275, 501, 12132, 2562, 294, 406, 455, 302, 281, 278, 283, 1217, 527, 332, 271, 912, 782, 440, 500, 261, 116, 284, 275, 5425, 302, 280, 406, 3013, 266, 449, 266, 284, 296, 1978, 271, 380, 642, 16378, 278, 7942, 302, 274, 289, 3645, 320, 266, 326, 304, 1439, 6178, 275, 1018, 271, 697, 345, 294, 330, 2377, 278, 280, 883, 305, 306, 3720, 266, 949, 330, 852, 271, 1373, 339, 1544, 275, 282, 225, 361, 266, 2353, 327, 2193, 290, 271, 9443, 266, 13457, 266, 354, 780, 5164, 296, 3251, 289, 3645, 4311, 596, 4311, 334, 301, 780, 327, 1426, 375, 10907, 266, 1088, 275, 5717, 275, 635, 305, 5843, 278, 294, 289, 637, 289, 6988, 296, 341, 1921, 302, 596, 3420, 266, 13802, 370, 324, 271, 279, 397, 266, 3525, 345, 294, 330, 438, 266, 3115, 340, 1179, 261, 116, 284, 275, 277, 10287, 275, 3676, 302, 281, 304, 558, 296, 2028, 338, 301, 2880, 266, 2119, 301, 531, 266, 319, 275, 501, 266, 1321, 271, 10271, 266, 530, 2631, 275, 15047, 26, 297, 278, 324, 271, 1392, 275, 282, 3901, 301, 705, 4939, 289, 3645, 356, 302, 297, 450, 1216, 553, 411, 289, 2571, 266, 746, 1925, 266, 274, 289, 4679, 411, 289, 4355, 6407, 274, 289, 718, 305, 6747, 266, 457, 426, 3358, 266, 746, 1179, 301, 320, 266, 274, 289, 1955, 426, 267, 14864, 266, 332, 271, 357, 271, 359, 2602, 271, 2926, 305, 6747, 266, 457, 426, 2906, 282, 305, 6747, 266, 1143, 356, 302, 297, 450, 456, 275, 1824, 273, 345, 294, 330, 381, 275, 279, 9399, 1591, 320, 266, 283, 619, 276, 5392, 327, 5456, 266, 889, 296, 12955, 301, 7998, 14431, 2094, 295, 275, 3242, 2094, 906, 301, 1819, 397, 266, 9399, 1591, 906, 301, 1754, 266, 332, 271, 380, 885, 459, 1385, 289, 478, 271, 814, 261, 116, 274, 275, 456, 301, 2531, 417, 277, 364, 271, 814, 261, 116, 320, 266, 656, 266, 539, 289, 274, 382, 271, 276, 2023, 266, 1397, 266, 656, 266, 539, 289, 274, 523, 480, 320, 266, 656, 266, 539, 289, 274, 409, 271, 1282, 275, 1065, 271, 3158, 327, 13024, 301, 461, 266, 456, 275, 3402, 390, 1224, 266, 4939, 289, 838, 275, 1531, 266, 283, 266, 2647, 417, 500, 275, 1065, 271, 1855, 302, 334, 301, 838, 275, 1561, 271, 2371, 6514, 275, 1564, 619, 455, 412, 2031, 651, 301, 1869, 384, 566, 8029, 396, 397, 266, 274, 301, 320, 266, 332, 271, 294, 417, 277, 4084, 266, 4760, 769, 266, 274, 289, 951, 330, 3194, 271, 465, 3148, 271, 294, 417, 2114, 2573, 301, 2005, 420, 266, 1179, 301, 280, 275, 1046, 273, 990, 16224, 370, 281, 278, 267, 266, 1232, 275, 282, 4112, 345, 294, 330, 521, 289, 3645, 320, 266, 276, 278, 883, 305, 306, 380, 307, 278, 279, 289, 18332, 1867, 2094, 906, 301, 1609, 558, 296, 2028, 1867, 400, 558, 296, 1556, 275, 368, 261, 116, 336, 271, 1867, 883, 305, 306, 3720, 266, 949, 330, 2114, 267, 266, 1232, 275, 282, 538, 406, 1253, 271, 1361, 420, 266, 4993, 275, 1018, 271, 5804, 301, 10164, 271, 813, 278, 297, 275, 4509, 327, 711, 340, 1902, 458, 264, 1202, 8652, 301, 501, 370, 288, 302, 596, 14431, 558, 296, 18548, 275, 4111, 301, 2435, 271, 310, 289, 3645, 274, 275, 730, 266, 1373, 278, 883, 305, 306, 324, 271, 2371, 273, 345, 294, 330, 1330, 289, 2460, 9131, 380, 371, 278, 512, 2064, 278, 297, 275, 1750, 454, 2947, 339, 1544, 275, 2587, 271, 2129, 568, 271, 297, 275, 4915, 340, 280, 275, 1831, 278, 333, 275, 282, 2]\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.cls_token_id)\n",
    "print(tokenizer.num_special_tokens_to_add())\n",
    "tokenized_text = tokenizer.tokenize(\"ฉันเคยเกือบพลาดสิ่งที่ดีที่สุดในชีวิต หากในวันที่ฉันล้มอยู่ ไม่มีหนึ่งใจของเธอ ฝันคงจบ หลายสิ่งที่ดีคงหมดทางได้เจอ หนึ่งกำลังใจที่ยิ่งใหญ่ ไม่ลืมได้เลย... ใครเคยมีแฟนที่กินอาหารไม่ถูกปากกันแล้วรู้สึกเสียความสุขไปอย่างนึงบ้างมั้ยครับ  ก่อนอื่นผมต้องบอกก่อนเลยว่าคนเราจะเลือกกินอาหารแบบไหนชอบแบบไหนเป็นเรื่องของความชอบส่วนตัวนะครับทุกคนมีสิทธิในการเลือกของที่ชอบและไม่ชอบอยู่แล้ว แต่ผมรู้สึกว่าตอนนี้ผมกำลังประสบปัญหาที่ดูเหมือนจะเล็กแต่กลายเป็นว่ามันค่อนข้างใหญ่ ผมคบกับแฟนมา6ปีแล้วครับ ผมเป็นคนชอบกินอาหารญี่ปุ่นและปลาดิบแต่แฟนผมไม่กินปลาดิบเลย ผมอยากกินบุฟเฟ่เนื้อแต่แฟนผมก็ไม่กินเนื้อ เราเลยไม่ได้เข้าทานร้านบุฟเฟ่เนื้อและบุฟเฟ่อาหารญี่ปุ่นกันเพราะรู้สึกลัวแฟนผมทานไม่คุ้ม และเรื่องใหญ่เลยคือผมเป็นคนชอบทานอาหารรสจัดและรสเผ็ดมาก แต่แฟนผมทานเผ็ดไม่ได้เลยเวลาเราไปกินส้มตำกันก็จะสั่ง ส้มตำไม่ใส่พริก ต้มแซ่บไม่ใส่พริก ลาบไม่ใส่พริก ร้านกับข้าวอื่นๆก็เช่นกันแฟนผมจะไม่ชอบกินผักไม่ค่อยสั่งกับข้าวที่เป็นผักแล้วผมชอบผักบุ้งทอดกรอบ เห็ดหอมสดทอดมาก แต่ก็ไม่ได้สั่งเพราะว่าเธอไม่กินถึงเค้าจะบอกให้สั่งเลยๆก็เถอะแต่ผมก็ยังเกรงใจเธออยู่ดีอ่ะครับ ผมรู้สึกกินอาหารไม่มีความสุขเลยชีวิตผมขาดรสเผ็ดไปเหมือนจะขาดใจเหมือนมันทำให้ขาดความสุขไปอย่างนึงเลยอ่ะครับ ยิ่งถ้าเราแต่งงานกันแล้วผมก็อาจจะต้องมีปัญหาเรื่องนี้มากขึ้น พอผมเห็นคู่ที่ชอบทานอาหารเหมือนๆกันเห็นเค้ากินอาหารกันอย่างมีความสุขแล้วผมรู้สึกอิจฉามากๆเลย มีใครเคยมีปัญหาแบบผมมั้ยครับแล้วจะแก้ปัญหานี้ยังไงดีครับ\")\n",
    "print(f\"{tokenized_text} with length {len(tokenized_text)}\")\n",
    "tokenized_text = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
    "print(f\"{tokenized_text} with length {len(tokenized_text)}\")\n",
    "print(tokenizer.build_inputs_with_special_tokens(tokenized_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Constructing tokenizer wrapper based on [@theblackcat102 #259](https://github.com/huggingface/tokenizers/issues/259#issuecomment-625905930)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class RobertaTokenizerFast(PreTrainedTokenizerFast):\n",
    "\n",
    "#     def __init__(\n",
    "#         self,\n",
    "#         vocab_file,\n",
    "#         merges_file,\n",
    "#         bos_token=\"<s>\",\n",
    "#         eos_token=\"</s>\",\n",
    "#         sep_token=\"</s>\",\n",
    "#         cls_token=\"<s>\",\n",
    "#         unk_token=\"<unk>\",\n",
    "#         pad_token=\"<pad>\",\n",
    "#         mask_token=\"<mask>\",\n",
    "#         **kwargs\n",
    "#     ):\n",
    "#         super().__init__(\n",
    "#             ByteLevelBPETokenizer(vocab_file=vocab_file,\n",
    "#                                     merges_file=merges_file,\n",
    "#                                   lowercase=True,                 # adds lowercase to normalizer\n",
    "#                                   unicode_normalizer=\"nfkc\",      # adds unicode normalizer nfkc to normalizer\n",
    "#                                   add_prefix_space=True,          # add space to the first word\n",
    "#                                  ),\n",
    "#              bos_token=bos_token,\n",
    "#             eos_token=eos_token,\n",
    "#             unk_token=unk_token,\n",
    "#             sep_token=sep_token,\n",
    "#             cls_token=cls_token,\n",
    "#             pad_token=pad_token,\n",
    "#             mask_token=mask_token,\n",
    "#             **kwargs,\n",
    "#         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizer3 = RobertaTokenizerFast(vocab_file= \"./thwiki-seniorproj-bytebpe-30522/vocab.json\",\n",
    "#                                  merges_file=\"./thwiki-seniorproj-bytebpe-30522/merges.txt\")\n",
    "# print(tokenizer3.num_special_tokens_to_add())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizer2 = Tokenizer.from_file(\"./thwiki-seniorproj-bytebpe-30522.tokenizer.json\")\n",
    "# tokenizer2.num_special_tokens_to_add(is_pair=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizer2.normalize(\"asdpo qweasd\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building our dataset\n",
    "\n",
    "Build it with `from torch.utils.data.dataset import Dataset` just like [TextDataset](https://github.com/huggingface/transformers/blob/448c467256332e4be8c122a159b482c1ef039b98/src/transformers/data/datasets/language_modeling.py) and [LineByLineTextDataset](https://github.com/huggingface/transformers/blob/448c467256332e4be8c122a159b482c1ef039b98/src/transformers/data/datasets/language_modeling.py#L78)\n",
    "\n",
    "Note: Training with multiple files is currently not supported [issue/3445](https://github.com/huggingface/transformers/issues/3445)\n",
    "\n",
    "padding documentation [link](https://github.com/huggingface/tokenizers/blob/master/bindings/python/tokenizers/implementations/base_tokenizer.py#L52)\n",
    "\n",
    "Potential Improvements\n",
    "- การทำให้ Dataset นั้น dynamically tokenize + dynamically open file : ตอนนี้เวลาทำ Dataset จาก torch.utils.data.dataset จะทำการ tokenize เลยตอนอยู่ใน constructor  , กำลังคิดว่าถ้าเกิดว่า Data ใหญ่มากๆ อาจจะไม่เหมาะสมกับการทำแบบนี้  เพราะว่า Ram จะต้องมีขนาดเท่าๆกับ data ที่เราใส่เข้าไป  ซึ่งเป็นไปได้ยากหาก Data มีขนาดใหญ่มากๆ   ผมได้ทำการ Search ดูแล้วก็พบว่าจาก Discussion Forum ของ Pytorch: https://discuss.pytorch.org/t/how-to-use-a-huge-line-corpus-text-with-dataset-dataloader/30872 \n",
    "Option1: ใช้ pd.Dataframe ในการเปิด File แบบ small chunks of data https://discuss.pytorch.org/t/data-processing-as-a-batch-way/14154/4?u=ptrblck\n",
    "Option2: ใช้ byte Offsets จากไฟล์ใหญ่ๆเพื่อที่จะ lookup .seek(): https://github.com/pytorch/text/issues/130#issuecomment-510412877\n",
    "More Examples: https://github.com/pytorch/text/blob/master/torchtext/datasets/unsupervised_learning.py , https://github.com/pytorch/text/blob/a5880a3da7928dd7dd529507eec943a307204de7/examples/text_classification/iterable_train.py#L169-L214"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "logger = logging.getLogger(__name__)\n",
    "class TextDatasetParallel(Dataset):\n",
    "    \"\"\"\n",
    "    This will be superseded by a framework-agnostic approach\n",
    "    soon.\n",
    "    \"\"\"         \n",
    "    def __init__(self, tokenizer: PreTrainedTokenizer, sample_path: [], block_size: int, overwrite_cache=False,\n",
    "                num_processes=8, cached_directory = \"/workdir/Code/bma_transformer_model/data/cached_data\"):\n",
    "        # assert os.path.isfile(file_path)\n",
    "        # For Loop MultiFile\n",
    "        self.examples = []\n",
    "        self.sample_path = sample_path\n",
    "#         print(f\"THIS IS SAMPLE PATH {sample_path}\")\n",
    "        self.tokenizer = tokenizer\n",
    "        \n",
    "        # Set block size to be the blocksize-special tokens\n",
    "        self.block_size = block_size - tokenizer.num_special_tokens_to_add(pair=False)\n",
    "        \n",
    "        self.overwrite_cache = overwrite_cache\n",
    "        self.cached_directory = cached_directory\n",
    "        if not os.path.exists(cached_directory):\n",
    "            os.makedirs(cached_directory)\n",
    "        \n",
    "        # Multiprocess for getting examples\n",
    "        with Pool(processes=num_processes) as p:\n",
    "            self.examples = list(tqdm.tqdm(p.imap(self.load_data_tokenized, self.sample_path), total=len(self.sample_path)))\n",
    "        # Convert from 3d list to 2d \n",
    "        # self.examples from [[[3], [4]], [[5], [6]], [[7], [8]]] => [[3], [4], [5], [6], [7], [8]]\n",
    "        self.examples = [each_batch for each_file in self.examples for each_batch in each_file]\n",
    "        \n",
    "\n",
    "    def load_data_tokenized(self, file_path):\n",
    "#         print(f\"I AM DOING {file_path}\")\n",
    "        directory, filename = os.path.split(file_path)\n",
    "        cached_features_file = os.path.join(\n",
    "            self.cached_directory, f\"cached_lm_{tokenizer.__class__.__name__}_{str(self.block_size)}_{filename}\",\n",
    "        )\n",
    "\n",
    "        # Make sure only the first process in distributed training processes the dataset,\n",
    "        # and the others will use the cache.\n",
    "        lock_path = cached_features_file + \".lock\"\n",
    "        with FileLock(lock_path):\n",
    "            if os.path.exists(cached_features_file) and not self.overwrite_cache:\n",
    "                start = time.time()\n",
    "                with open(cached_features_file, \"rb\") as handle:\n",
    "                    temp_examples = pickle.load(handle)\n",
    "                logger.info(\n",
    "                    f\"Loading features from cached file {cached_features_file} [took %.3f s]\", time.time() - start\n",
    "                )\n",
    "            else:\n",
    "                temp_examples = []\n",
    "                with open(file_path, encoding=\"utf-8\") as f:\n",
    "                    text = f.read()\n",
    "#                 print(\"I finished reading \", file_path)\n",
    "                tokenized_text = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text))\n",
    "#                 print(\"I finished tokenizing \", file_path)\n",
    "                for i in range(0, len(tokenized_text) - self.block_size + 1, self.block_size):  # Truncate in block of block_size\n",
    "                    temp_examples.append(\n",
    "                        tokenizer.build_inputs_with_special_tokens(tokenized_text[i : i + self.block_size])\n",
    "                    )\n",
    "#                     if i%20 == 0:\n",
    "#                         print(\"I finished special tok \", file_path)\n",
    "#                 print(\"I finished every tokenizing \", file_path)\n",
    "                # Note that we are losing the last truncated example here for the sake of simplicity (no padding)\n",
    "                # If your dataset is small, first you should loook for a bigger one :-) and second you\n",
    "                # can change this behavior by adding (model specific) padding.\n",
    "\n",
    "                start = time.time()\n",
    "                with open(cached_features_file, \"wb\") as handle:\n",
    "                    pickle.dump(temp_examples, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "                logger.info(\n",
    "                    \"Saving features into cached file %s [took %.3f s]\", cached_features_file, time.time() - start\n",
    "                )\n",
    "        return temp_examples\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.examples)\n",
    "\n",
    "    def __getitem__(self, i) -> torch.Tensor:\n",
    "        return torch.tensor(self.examples[i], dtype=torch.long)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenizer from Pretrained copied from SimpleTransformers [link](https://github.com/ThilinaRajapakse/simpletransformers/blob/master/simpletransformers/language_modeling/language_modeling_model.py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1409/1409 [18:21<00:00,  1.28it/s]  \n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# logging.basicConfig(level=logging.WARN)\n",
    "dataset = TextDatasetParallel(tokenizer, \n",
    "                              sample_path=list(map(str, ALL_FILES)), \n",
    "#                               sample_path=list(map(str, GURU_CRAWLER_FILES)), \n",
    "                              block_size=512, \n",
    "                              cached_directory= \"/workdir/cached_data\",\n",
    "                              overwrite_cache=False, # make sure this is false when you have cache!!\n",
    "                              num_processes=64,\n",
    "                             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cached_directory= \"/workdir/cached_data\"\n",
    "# def load_data_tokenized(file_path):\n",
    "#     directory, filename = os.path.split(file_path)\n",
    "#     cached_features_file = os.path.join(\n",
    "#         cached_directory, f\"cached_lm_something_{str(123)}_{filename}\",\n",
    "#     )\n",
    "#     temp_examples = []\n",
    "#     with open(file_path, encoding=\"utf-8\") as f:\n",
    "#         text = f.read()\n",
    "#     print(\"I finished reading \", file_path)\n",
    "#     tokenized_text = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text[:1000]))\n",
    "#     print(\"I finished tokenizing \", tokenized_text[:100])\n",
    "#     return f\"{file_path}+123\"\n",
    "# # Multiprocess for getting examples\n",
    "# with Pool(processes=2) as p:\n",
    "#     examples = list(tqdm.tqdm(p.imap(load_data_tokenized, list(map(str, GURU_CRAWLER_FILES))), total=len(list(map(str, GURU_CRAWLER_FILES)))))\n",
    "# print(examples)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'dataset' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-9577e75d4bf6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'dataset' is not defined"
     ]
    }
   ],
   "source": [
    "dataset.__len__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "512"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset.__getitem__(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([    0,  2398,   289, 12660,   345,  6648,   295,   275,   836,   275,\n",
       "        11206, 10813,   278,   273,   278,   306,   275,   851,   317,   203,\n",
       "          295,   275,   836,   275, 11206, 10813,   278,   273,   278,   306,\n",
       "          275,   851,   317,   371,   278,  2126,   338,   296,  1208,   339,\n",
       "        15232,   289, 12660,   345,  6648,   485,  3752,   508,   294,   312,\n",
       "          613,   275,  8811,   296,  1208,   339,   277,   401,    18,   376,\n",
       "           18,  1884,    13,  1640,   278,  2398,   289, 12660,   345,  6648,\n",
       "          454,  1383,   266,  1470, 12168,   296,  1208,   339,   952,   271,\n",
       "          485,   616,  3713,   339,  1217,  6299,  1861,   417,  3466,  1603,\n",
       "         8517,   345,  6648,   515,   203,   372,   275,  2216,   302,  1371,\n",
       "          594,   302,  2591, 17573,  2020,   278,   401,    18,   376,    18,\n",
       "         1639,  3227,   604,   271,   276,   278,  1618,   275, 16404,   296,\n",
       "         1208,   339, 15232,   289, 12660,   345,  6648,   334,   301,   829,\n",
       "          339,  1651,   429,   327,   572,   275,   368,   302,  1101,  4527,\n",
       "          289,  4218,   401,    18,   376,    18,  1639,  8438,  7459,   296,\n",
       "         1208,   339,  1070,   339,   535,   340,   334,   301,   264,   660,\n",
       "         2148,   296,  1208,   339,   455,  1684,   271, 17587,   339,  1651,\n",
       "          350,   333,   339,  1445,   278,   425,  5754,     6,  9497,   275,\n",
       "          836,   275, 11206, 10813,   278,   273,   278,   306,   275,   851,\n",
       "          317,   276,   278,  2398,   289, 12660,   345,  6648,  9556,   391,\n",
       "          296,   267,   877,   276,   406,   277,   586,   377,   317,   329,\n",
       "          347,   304,   203,  2398,   289, 12660,   345,  6648,   295,   275,\n",
       "          836,   275,  1785,  9322,   203,   295,   275,   836,   275,  1785,\n",
       "         9322,   371,   278,  2126,   338,   296,  1208,   339, 15232,   289,\n",
       "        12660,   345,  6648,   485,  3752,   508,   294,   312,   613,   275,\n",
       "         8811,   296,  1208,   339,   277,   401,    18,   376,    18,  1884,\n",
       "           13,  1640,   278,  2398,   289, 12660,   345,  6648,   454,  1383,\n",
       "          266,  1470, 12168,   296,  1208,   339,   952,   271,   485,   616,\n",
       "         3713,   339,  1217,  6299,  1861,   417,  3466,  1603,  8517,   345,\n",
       "         6648,   515,   203,   372,   275,  2216,   302,  1371,   594,   302,\n",
       "         2591, 17573,  2020,   278,   401,    18,   376,    18,  1639,  3227,\n",
       "          604,   271,   276,   278,  1618,   275, 16404,   296,  1208,   339,\n",
       "        15232,   289, 12660,   345,  6648,   334,   301,   829,   339,  1651,\n",
       "          429,   327,   572,   275,   368,   302,  1101,  4527,   289,  4218,\n",
       "          401,    18,   376,    18,  1639,  8438,  7459,   296,  1208,   339,\n",
       "         1070,   339,   535,   340,   334,   301,   264,   660,  2148,   296,\n",
       "         1208,   339,   455,  1684,   271, 17587,   339,  1651,   350,   333,\n",
       "          339,  1445,   278,   425,  5754,     6,   569,  5092,   339,   744,\n",
       "         6847,   315,   275,   773,   301,   985,   278,  7691,   312,  1908,\n",
       "         1664,   440, 16316,   275,   836,   275,  1804,   278,  3770,   203,\n",
       "          297,   278,   401,    18,   376,    18,   772,   965,   401,  6847,\n",
       "         1152,  2456,   306,   458,   756,   301,   747,   275,   836,   275,\n",
       "          281,  3291,   271,   276,   278,  2148,   296,  1208,   339, 15232,\n",
       "          289, 12660,   345,  6648,  4937,   275,   836,   275,  1973,   301,\n",
       "          829,   339,  1651,   508,  6988,   296,  1208,   339,   277,   401,\n",
       "           18,   376,    18,   772,  2512,    13,  1997,   278,  2398,   289,\n",
       "        12660,   345,  6648,   307,   305,  7676,   391,   296,   267,  7435,\n",
       "          308, 19523,   301,   274,   382,   275,   264,  4626,   277,    16,\n",
       "          877,  9728,   275,   851,   317,   433,   275,   846,   305,   307,\n",
       "         2944,   271,  2883,   312,  1032,   419,   278,   460,   275,  1252,\n",
       "          289,   290,  5311,  1977,   275,  8981,   203,   274,  2858,   289,\n",
       "          267,     2])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.__getitem__(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# print(text[:1000])\n",
    "# print(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text[:1000])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__For text[:100000]__  \n",
    "\n",
    "Rust implementation\n",
    ">\n",
    "\n",
    "Python\n",
    ">CPU times: user 900 ms, sys: 4 ms, total: 904 ms\n",
    "Wall time: 903 ms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text[:100000]))\n",
    "# None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__For text[:1000000]__  \n",
    "\n",
    "Rust implementation\n",
    ">\n",
    "\n",
    "Python\n",
    ">CPU times: user 7.27 s, sys: 40 ms, total: 7.31 s\n",
    "Wall time: 7.31 s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text[:1000000]))\n",
    "# None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__For text[:3000000]__  \n",
    "\n",
    "Rust implementation\n",
    ">CPU times: user 6.38 s, sys: 328 ms, total: 6.7 s  \n",
    "Wall time: 5.18 s\n",
    "\n",
    "Python\n",
    ">CPU times: user 15.5 s, sys: 72 ms, total: 15.6 s  \n",
    "Wall time: 15.6 s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text[:3000000]))\n",
    "# None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__For text[:8000000]__  \n",
    "\n",
    "Rust implementation\n",
    ">\n",
    "\n",
    "Python\n",
    ">CPU times: user 36.1 s, sys: 340 ms, total: 36.4 s  \n",
    "Wall time: 36.4 s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text[:8000000]))\n",
    "# None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i_batch, sample_batched in enumerate(dataloader):\n",
    "#     print(i_batch, sample_batched)\n",
    "#     oumodel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizer = CharBPETokenizer(vocab_file='vocab.json',merges_file ='merges.txt' )\n",
    "# no_accent_strip = BertNormalizer(strip_accents=False)\n",
    "# tokenizer._tokenizer.normalizer = no_accent_strip\n",
    "# tokenizer._tokenizer.post_processor = BertProcessing(\n",
    "#     (\"</s>\", tokenizer.token_to_id(\"</s>\")),\n",
    "#     (\"<s>\", tokenizer.token_to_id(\"<s>\")),\n",
    "# )\n",
    "\n",
    "# input_ids = torch.tensor(tokenizer.encode(u\"สวัสดีครับ ผมชื่อไนท์ ตอนนี้ก็เป็นเวลาที่ผมต้องไปโรงเรียนแล้ว  นี่คือการเว้นวรรคสองทีครับ  จะได้ออกเป็นสอง Spaces\").ids).unsqueeze(0)\n",
    "# print(input_ids)\n",
    "# outputs = model(input_ids, labels=input_ids)\n",
    "# print(outputs)\n",
    "# loss, prediction_scores = outputs[:2]\n",
    "# print(loss, prediction_scores.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset.__getitem__(1).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input_ids = torch.tensor(tokenizer.encode(\"Hello, my dog is cute\", add_special_tokens=True)).unsqueeze(0)  # Batch size 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# from transformers import TextDataset, LineByLineTextDataset\n",
    "\n",
    "# # dataset = LineByLineTextDataset(\n",
    "# #     tokenizer=pretrain_tokenizer,\n",
    "# #     file_path=\"../data/text/AA/wiki_01\",\n",
    "# #     block_size=128,\n",
    "# # )\n",
    "\n",
    "# dataset = TextDataset(\n",
    "#     tokenizer=pretrain_tokenizer,\n",
    "#     file_path=\"../data/text/AA/wiki_01\",\n",
    "#     block_size=128,\n",
    "# )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one_doc = list(Path(\"../data/text/AA/\").glob(\"wiki*\"))[0].read_text(encoding=\"utf-8\").splitlines()\n",
    "# tokenizer = Tokenizer.from_file(\"./thwiki-sentencepiecebpe.tokenizer.json\")\n",
    "# tokenizer.encode_batch(one_doc[:8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one_doc = list(Path(\"../data/text/AA/\").glob(\"wiki*\"))[0].read_text(encoding=\"utf-8\").splitlines()\n",
    "# tokenizer = RobertaTokenizerFast(vocab_file='vocab.json',merges_file ='merges.txt', max_len=512)\n",
    "# tokenizer.batch_encode_plus(one_doc[:8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# print(tokenizer.encode_batch(one_doc[:8])[5].tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one_doc[:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForLanguageModeling\n",
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=tokenizer, mlm=True, mlm_probability=0.15\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transfomers Trainer [link](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py#L133)\n",
    "\n",
    "```python\n",
    "class Trainer:\n",
    "    \"\"\"\n",
    "    Trainer is a simple but feature-complete training and eval loop for PyTorch,\n",
    "    optimized for Transformers.\n",
    "    Args:\n",
    "        prediction_loss_only:\n",
    "            (Optional) in evaluation and prediction, only return the loss\n",
    "    \"\"\"\n",
    "    def __init__(\n",
    "        self,\n",
    "        model: PreTrainedModel,\n",
    "        args: TrainingArguments,\n",
    "        data_collator: Optional[DataCollator] = None,\n",
    "        train_dataset: Optional[Dataset] = None,\n",
    "        eval_dataset: Optional[Dataset] = None,\n",
    "        compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None,\n",
    "        prediction_loss_only=False,\n",
    "        tb_writer: Optional[\"SummaryWriter\"] = None,\n",
    "        optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = None,\n",
    "```\n",
    "\n",
    "[TrainingArguments](https://github.com/huggingface/transformers/blob/master/src/transformers/training_args.py#L33) is referenced here. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./Roberta\",\n",
    "    overwrite_output_dir=False,  #\"Use this to continue training if output_dir points to a checkpoint directory.\"\n",
    "    \n",
    "#     fp16=True,\n",
    "#     fp16_opt_level='O0',\n",
    "    \n",
    "    \n",
    "    do_train=True, #Whether to run training.\n",
    "#     do_eval=True, #Whether to run eval on the dev set.\n",
    "#     do_predict=True, # Whether to run predictions on the test set.\n",
    "    \n",
    "    num_train_epochs=200, # Total number of training epochs to perform.\n",
    "    \n",
    "    \n",
    "    per_device_train_batch_size=10, # Batch size per GPU/TPU core/CPU for training.\n",
    "#     per_device_eval_batch_size=256, # Batch size per GPU/TPU core/CPU for evaluation.\n",
    "    \n",
    "    learning_rate=5e-5,  #The initial learning rate for Adam.\n",
    "    weight_decay=0.0,\n",
    "    max_grad_norm=1.0,\n",
    "    adam_epsilon=1e-8, #Epsilon for Adam optimizer.\n",
    "    \n",
    "    #Logging\n",
    "#     logging_dir='', default_logdir -> return os.path.join(\"runs\", current_time + \"_\" + socket.gethostname())\n",
    "    logging_first_step= True,\n",
    "    logging_steps = 500,\n",
    "    \n",
    "    save_steps=10_000,  #Save checkpoint every X updates steps.\n",
    "    save_total_limit=2, #\"Limit the total amount of checkpoints. Deletes the older checkpoints in the output_dir. Default is unlimited checkpoints\n",
    "    \n",
    "    seed=42,\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    data_collator=data_collator,\n",
    "    train_dataset=dataset,\n",
    "#     eval_dataset=val_dataset\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b141d24a0aa4a0cbfbdaacdec90ef12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=200.0, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d2a6cb436c24c938d4c1def4bc9a280",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=233892.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/nn/parallel/_functions.py:61: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:200: UserWarning: Please also save or load the state of the optimzer when saving or loading the scheduler.\n",
      "  warnings.warn(SAVE_STATE_WARNING, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.save_model(\"./Roberta_Final\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}