train_acoustic_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c4f26172",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import json\n",
    "import os\n",
    "import shutil\n",
    "\n",
    "from datetime import datetime\n",
    "import IPython.display as ipd\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.options.display.max_columns = 100\n",
    "pd.options.display.max_colwidth = 200\n",
    "\n",
    "import datasets as hfd\n",
    "\n",
    "import transformers\n",
    "from transformers import (\n",
    "    Wav2Vec2CTCTokenizer, Wav2Vec2FeatureExtractor, Wav2Vec2Processor,\n",
    "    Wav2Vec2ForCTC,\n",
    "    TrainingArguments, Trainer\n",
    ")\n",
    "\n",
    "import torch\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a9b94978",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.data_collator import DataCollatorCTCWithPadding\n",
    "from src.metrics import WerMetric, parse_w2v2_predictions_batched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "96686227",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logging_format_str = '%(asctime)s:%(name)s:%(levelname)s:%(message)s'\n",
    "logging.basicConfig(format=logging_format_str, level=logging.WARNING)\n",
    "logger = logging.getLogger('STT')\n",
    "logger.setLevel(level=logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54815137",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5751538f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# DATA_ROOT_DP = os.environ['DATA_HOME']\n",
    "SSD_DATA_ROOT_DP = os.environ['SSD_DATA_HOME']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4fcf9f67",
   "metadata": {},
   "outputs": [],
   "source": [
    "CV_PROCESSED_2_DP = f'{SSD_DATA_ROOT_DP}/datasets/cv-corpus-8.0-2022-01-19__be__processed__2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b5a7db00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_values', 'labels'],\n",
       "        num_rows: 314305\n",
       "    })\n",
       "    dev: Dataset({\n",
       "        features: ['input_values', 'labels'],\n",
       "        num_rows: 15803\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_values', 'labels'],\n",
       "        num_rows: 15801\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = hfd.load_from_disk(CV_PROCESSED_2_DP)\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d3350da1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # select subset of data\n",
    "# ds['train'] = ds['train'].select(range(300))\n",
    "# ds['dev'] = ds['dev'].select(range(100))\n",
    "# ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d8e6f2e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "daa8ec74",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "processor = Wav2Vec2Processor.from_pretrained('artifacts/processor')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d18f0b3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorCTCWithPadding(processor=processor, padding=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cdc05448",
   "metadata": {},
   "outputs": [],
   "source": [
    "wer_metric = WerMetric(processor=processor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1718159",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2fdb8add",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>'</th>\n",
       "      <th>i</th>\n",
       "      <th>а</th>\n",
       "      <th>б</th>\n",
       "      <th>в</th>\n",
       "      <th>г</th>\n",
       "      <th>д</th>\n",
       "      <th>е</th>\n",
       "      <th>ж</th>\n",
       "      <th>з</th>\n",
       "      <th>й</th>\n",
       "      <th>к</th>\n",
       "      <th>л</th>\n",
       "      <th>м</th>\n",
       "      <th>н</th>\n",
       "      <th>о</th>\n",
       "      <th>п</th>\n",
       "      <th>р</th>\n",
       "      <th>с</th>\n",
       "      <th>т</th>\n",
       "      <th>у</th>\n",
       "      <th>ф</th>\n",
       "      <th>х</th>\n",
       "      <th>ц</th>\n",
       "      <th>ч</th>\n",
       "      <th>ш</th>\n",
       "      <th>ы</th>\n",
       "      <th>ь</th>\n",
       "      <th>э</th>\n",
       "      <th>ю</th>\n",
       "      <th>я</th>\n",
       "      <th>ё</th>\n",
       "      <th>і</th>\n",
       "      <th>ў</th>\n",
       "      <th>|</th>\n",
       "      <th>[UNK]</th>\n",
       "      <th>[PAD]</th>\n",
       "      <th>&lt;s&gt;</th>\n",
       "      <th>&lt;/s&gt;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>36</td>\n",
       "      <td>37</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   '  i  а  б  в  г  д  е  ж   з   й   к   л   м   н   о   п   р   с   т   у  \\\n",
       "0  1  2  3  4  5  6  7  8  9  10  11  12  13  14  15  16  17  18  19  20  21   \n",
       "\n",
       "    ф   х   ц   ч   ш   ы   ь   э   ю   я   ё   і   ў  |  [UNK]  [PAD]  <s>  \\\n",
       "0  22  23  24  25  26  27  28  29  30  31  32  33  34  0     35     36   37   \n",
       "\n",
       "   </s>  \n",
       "0    38  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = processor.tokenizer.get_vocab()\n",
    "pd.Series(vocab).to_frame().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ccaaa1e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-01 18:10:52,698:STT:INFO:vocab_size: 39\n"
     ]
    }
   ],
   "source": [
    "vocab_size = len(processor.tokenizer.get_vocab())\n",
    "logger.info(f'vocab_size: {vocab_size}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45ac58f4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb2fda77",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "7a0ffd82",
   "metadata": {},
   "source": [
    "## prepare for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6bb8334c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['scheduler.pt', 'preprocessor_config.json', 'training_args.bin', 'pytorch_model.bin', 'scaler.pt', 'config.json', 'rng_state.pth', 'optimizer.pt', 'trainer_state.json']\n"
     ]
    }
   ],
   "source": [
    "pretrained_checkpoint_dp = 'train/run_3/2022-03-31-10-42-03/checkpoint-6480'\n",
    "print(os.listdir(pretrained_checkpoint_dp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8646384",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f5757aab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = Wav2Vec2ForCTC.from_pretrained(\n",
    "#     \"facebook/wav2vec2-base\",\n",
    "#     ctc_loss_reduction=\"mean\", \n",
    "#     pad_token_id=processor.tokenizer.pad_token_id,\n",
    "#     vocab_size=vocab_size\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "14ff5326",
   "metadata": {},
   "outputs": [],
   "source": [
    "# continue from checkpoint\n",
    "model = Wav2Vec2ForCTC.from_pretrained(pretrained_checkpoint_dp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35c0d291",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.freeze_feature_extractor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ca4ae4c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0907435",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9a1c5ab6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "will save train output to: \"train/2022-03-31-21-50-25\"\n",
      "will save train logs to: \"train/2022-03-31-21-50-25/logs\"\n"
     ]
    }
   ],
   "source": [
    "now_str = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')\n",
    "\n",
    "train_out_dp = os.path.join('train', now_str)\n",
    "os.makedirs(train_out_dp, exist_ok=True)\n",
    "print(f'will save train output to: \"{train_out_dp}\"')\n",
    "\n",
    "logs_dp = os.path.join(train_out_dp, 'logs')\n",
    "print(f'will save train logs to: \"{logs_dp}\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a88e0830",
   "metadata": {},
   "outputs": [],
   "source": [
    "steps = 1080\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=train_out_dp,\n",
    "    logging_dir=logs_dp,\n",
    "\n",
    "    num_train_epochs=2,\n",
    "    evaluation_strategy='steps',\n",
    "    save_strategy='steps',\n",
    "    logging_strategy='steps',\n",
    "    eval_steps=steps,\n",
    "    logging_steps=steps,\n",
    "    save_steps=steps,\n",
    "    save_total_limit=None,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model='wer',\n",
    "    greater_is_better=False,\n",
    "\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    fp16=True,\n",
    "    group_by_length=False,\n",
    "    gradient_checkpointing=True, \n",
    "    \n",
    "    learning_rate=2e-5,\n",
    "    weight_decay=0.001,\n",
    "    warmup_steps=1080,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9ea75887",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp half precision backend\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    data_collator=data_collator,\n",
    "    args=training_args,\n",
    "    compute_metrics=wer_metric.compute_metrics,\n",
    "    train_dataset=ds['train'],\n",
    "    eval_dataset=ds['dev'],\n",
    "    tokenizer=processor.feature_extractor,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f17cd38",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "1e2d21cd",
   "metadata": {},
   "source": [
    "## train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cc926e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e3e7127",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cde22e28",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "703d2fda",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "72334c3d",
   "metadata": {},
   "source": [
    "## evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a94810d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading model from: \"train/run_4/2022-03-31-21-50-25/checkpoint-12960/\"\n"
     ]
    }
   ],
   "source": [
    "finetuned_checkpoint_dp = 'train/run_4/2022-03-31-21-50-25/checkpoint-12960/'\n",
    "print(f'reading model from: \"{finetuned_checkpoint_dp}\"')\n",
    "model = Wav2Vec2ForCTC.from_pretrained(finetuned_checkpoint_dp)\n",
    "model.to('cuda');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "061d5554",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a39f5f55",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir='eval_output',\n",
    "    per_device_eval_batch_size=16,\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    data_collator=data_collator,\n",
    "    args=training_args,\n",
    "    compute_metrics=wer_metric.compute_metrics,\n",
    "    tokenizer=processor.feature_extractor,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "144a154d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "ef469458",
   "metadata": {},
   "source": [
    "### dev set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8778aabe",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_out = trainer.predict(ds['dev'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3999a926",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'test_loss': 0.11704583466053009,\n",
       " 'test_wer': 0.17611859752710188,\n",
       " 'test_runtime': 493.3495,\n",
       " 'test_samples_per_second': 32.032,\n",
       " 'test_steps_per_second': 0.669}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_out.metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7ccf4fa1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER on Dev set using Acoustic Model only: 0.1761\n"
     ]
    }
   ],
   "source": [
    "print(f\"WER on Dev set using Acoustic Model only: {pred_out.metrics['test_wer'] :.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c6b3119",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e24017d1",
   "metadata": {},
   "source": [
    "### test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2f640506",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Prediction *****\n",
      "  Num examples = 15801\n",
      "  Batch size = 48\n"
     ]
    }
   ],
   "source": [
    "pred_out = trainer.predict(ds['test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "078459cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'test_loss': 0.12578321993350983,\n",
       " 'test_wer': 0.18748834172728968,\n",
       " 'test_runtime': 595.2022,\n",
       " 'test_samples_per_second': 26.547,\n",
       " 'test_steps_per_second': 0.554}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_out.metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "901c0f52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER on Test set using Acoustic Model only: 0.187\n"
     ]
    }
   ],
   "source": [
    "print(f\"WER on Test set using Acoustic Model only: {pred_out.metrics['test_wer'] :.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42c2d132",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "4e9f3c37",
   "metadata": {},
   "source": [
    "### examples of model predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3dbe5696",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set(font_scale=1.1);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "12180160",
   "metadata": {},
   "outputs": [],
   "source": [
    "parsed_preds = parse_w2v2_predictions_batched(pred_out, processor=processor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3ad57eb6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15801, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pred_text</th>\n",
       "      <th>success</th>\n",
       "      <th>wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>і на гэтым кірунку нас чакаюць вялікія складанасці</td>\n",
       "      <td>і на гэтым кірунку нас чакаюць вялікія складанасці</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>быў разбураны пры вызваленні горада ад нямецкафашысцкіх захопнікаў</td>\n",
       "      <td>быў разбураны пры вызваленні горада ад нямецкаю фашызсцкіх захокнікаў</td>\n",
       "      <td>0</td>\n",
       "      <td>0.375000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>далейшае супрацоўніцтва паміж нямецкім і савецкім прыняло форму абмену польскіх ваеннапалонных</td>\n",
       "      <td>далейшые супрацоўніцтва мічнямецкім савецкім пыня фораму абменну польскіх дваеннапогам</td>\n",
       "      <td>0</td>\n",
       "      <td>0.727273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>цвіценне працягваецца з мая да позняй восені</td>\n",
       "      <td>цвіценне працягваецца з мая да позняй восені</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>таксама пройдзе канферэнцыя фестываль і мастацкая выстава</td>\n",
       "      <td>таксама пройдзе канферэнцыя фестываль і мастацкая выстава</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                             text  \\\n",
       "0                                              і на гэтым кірунку нас чакаюць вялікія складанасці   \n",
       "1                              быў разбураны пры вызваленні горада ад нямецкафашысцкіх захопнікаў   \n",
       "2  далейшае супрацоўніцтва паміж нямецкім і савецкім прыняло форму абмену польскіх ваеннапалонных   \n",
       "3                                                    цвіценне працягваецца з мая да позняй восені   \n",
       "4                                       таксама пройдзе канферэнцыя фестываль і мастацкая выстава   \n",
       "\n",
       "                                                                                pred_text  \\\n",
       "0                                      і на гэтым кірунку нас чакаюць вялікія складанасці   \n",
       "1                   быў разбураны пры вызваленні горада ад нямецкаю фашызсцкіх захокнікаў   \n",
       "2  далейшые супрацоўніцтва мічнямецкім савецкім пыня фораму абменну польскіх дваеннапогам   \n",
       "3                                            цвіценне працягваецца з мая да позняй восені   \n",
       "4                               таксама пройдзе канферэнцыя фестываль і мастацкая выстава   \n",
       "\n",
       "   success       wer  \n",
       "0        1  0.000000  \n",
       "1        0  0.375000  \n",
       "2        0  0.727273  \n",
       "3        1  0.000000  \n",
       "4        1  0.000000  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results = pd.DataFrame(parsed_preds)[['text', 'pred_text']]\n",
    "print(df_results.shape)\n",
    "\n",
    "df_results['success'] = (df_results['text'] == df_results['pred_text']).astype('int')\n",
    "df_results['wer'] = df_results.apply(\n",
    "    lambda row: \n",
    "    wer_metric.wer_metric.compute(\n",
    "        predictions=[row['pred_text']],\n",
    "        references=[row['text']]\n",
    "    ),\n",
    "    axis=1\n",
    ")\n",
    "\n",
    "df_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9447dad3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fraction</th>\n",
       "      <th>abs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.633124</td>\n",
       "      <td>10004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.366876</td>\n",
       "      <td>5797</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fraction    abs\n",
       "0  0.633124  10004\n",
       "1  0.366876   5797"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success_stats = df_results['success'].value_counts(dropna=False, normalize=True)\n",
    "success_stats = success_stats.to_frame('fraction').join(\n",
    "    df_results['success'].value_counts(dropna=False).rename('abs')\n",
    ")\n",
    "success_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ec0ffe68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate of fully recognized clips from Test set using Acoustic Model only: 36.688%\n"
     ]
    }
   ],
   "source": [
    "print(f'Rate of fully recognized clips from Test set using Acoustic Model only: '\n",
    "      f\"{success_stats.loc[1, 'fraction'] :.3%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c67a5e0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "84d32f50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pred_text</th>\n",
       "      <th>success</th>\n",
       "      <th>wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8877</th>\n",
       "      <td>дрэвы маюць гладкую светлашэрую кару</td>\n",
       "      <td>трэба мач вадкую свет лашэрую каромну</td>\n",
       "      <td>0</td>\n",
       "      <td>1.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13492</th>\n",
       "      <td>азаранка здабыла дзевятнаццаты тытул у кар'еры</td>\n",
       "      <td>а зарнка зда была дзевятна стытытул і кар'еры</td>\n",
       "      <td>0</td>\n",
       "      <td>1.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1988</th>\n",
       "      <td>урэшце гнілую кукурузу скасілі напачатку гэтага году</td>\n",
       "      <td>турэшце гнізую укурозу з каселі на пачатку гэтага годы</td>\n",
       "      <td>0</td>\n",
       "      <td>1.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1651</th>\n",
       "      <td>уваход на сустрэчу свабодны</td>\n",
       "      <td>уваход ны сустрэча сва бодны</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10355</th>\n",
       "      <td>было хоць вушы затыкай</td>\n",
       "      <td>было хоць у нушы за такай</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       text  \\\n",
       "8877                   дрэвы маюць гладкую светлашэрую кару   \n",
       "13492        азаранка здабыла дзевятнаццаты тытул у кар'еры   \n",
       "1988   урэшце гнілую кукурузу скасілі напачатку гэтага году   \n",
       "1651                            уваход на сустрэчу свабодны   \n",
       "10355                                было хоць вушы затыкай   \n",
       "\n",
       "                                                    pred_text  success  \\\n",
       "8877                    трэба мач вадкую свет лашэрую каромну        0   \n",
       "13492           а зарнка зда была дзевятна стытытул і кар'еры        0   \n",
       "1988   турэшце гнізую укурозу з каселі на пачатку гэтага годы        0   \n",
       "1651                             уваход ны сустрэча сва бодны        0   \n",
       "10355                               было хоць у нушы за такай        0   \n",
       "\n",
       "            wer  \n",
       "8877   1.200000  \n",
       "13492  1.166667  \n",
       "1988   1.142857  \n",
       "1651   1.000000  \n",
       "10355  1.000000  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.sort_values('wer', ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d42317b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pred_text</th>\n",
       "      <th>success</th>\n",
       "      <th>wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>і на гэтым кірунку нас чакаюць вялікія складанасці</td>\n",
       "      <td>і на гэтым кірунку нас чакаюць вялікія складанасці</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8475</th>\n",
       "      <td>трэба каб і на стале было</td>\n",
       "      <td>трэба каб і на стале было</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8474</th>\n",
       "      <td>мы павінны сёння ж выехаць</td>\n",
       "      <td>мы павінны сёння ж выехаць</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8471</th>\n",
       "      <td>у архіве захоўваецца некалькі рэдкіх рукапісаў</td>\n",
       "      <td>у архіве захоўваецца некалькі рэдкіх рукапісаў</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8465</th>\n",
       "      <td>мне гэта падавалася глупствам</td>\n",
       "      <td>мне гэта падавалася глупствам</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  \\\n",
       "0     і на гэтым кірунку нас чакаюць вялікія складанасці   \n",
       "8475                           трэба каб і на стале было   \n",
       "8474                          мы павінны сёння ж выехаць   \n",
       "8471      у архіве захоўваецца некалькі рэдкіх рукапісаў   \n",
       "8465                       мне гэта падавалася глупствам   \n",
       "\n",
       "                                               pred_text  success  wer  \n",
       "0     і на гэтым кірунку нас чакаюць вялікія складанасці        1  0.0  \n",
       "8475                           трэба каб і на стале было        1  0.0  \n",
       "8474                          мы павінны сёння ж выехаць        1  0.0  \n",
       "8471      у архіве захоўваецца некалькі рэдкіх рукапісаў        1  0.0  \n",
       "8465                       мне гэта падавалася глупствам        1  0.0  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.sort_values('wer', ascending=True).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ebd058d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "852ff4c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pred_text</th>\n",
       "      <th>success</th>\n",
       "      <th>wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>толькі ў раёне таксама нашы людзі вясковыя</td>\n",
       "      <td>толькі ў раёне таксама наш людзі вясковыя</td>\n",
       "      <td>0</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9532</th>\n",
       "      <td>у яго нават ёсць сольныя выступы</td>\n",
       "      <td>у яго нават ёсць сёльныя выступы</td>\n",
       "      <td>0</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9533</th>\n",
       "      <td>на ўзвышшы што ў цэнтры пляцоўкі стаяла абарончая вежа</td>\n",
       "      <td>наўзвышшы што ў цэнтры пляцоўкі стаяла абарончая вежа</td>\n",
       "      <td>0</td>\n",
       "      <td>0.222222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12305</th>\n",
       "      <td>шлюбам папярэднічалі перагаворы паміж сем'ямі</td>\n",
       "      <td>шлюбам папярэднічалі перагаворы паміж сем'ямі</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                         text  \\\n",
       "12                 толькі ў раёне таксама нашы людзі вясковыя   \n",
       "9532                         у яго нават ёсць сольныя выступы   \n",
       "9533   на ўзвышшы што ў цэнтры пляцоўкі стаяла абарончая вежа   \n",
       "12305           шлюбам папярэднічалі перагаворы паміж сем'ямі   \n",
       "\n",
       "                                                   pred_text  success  \\\n",
       "12                 толькі ў раёне таксама наш людзі вясковыя        0   \n",
       "9532                        у яго нават ёсць сёльныя выступы        0   \n",
       "9533   наўзвышшы што ў цэнтры пляцоўкі стаяла абарончая вежа        0   \n",
       "12305          шлюбам папярэднічалі перагаворы паміж сем'ямі        1   \n",
       "\n",
       "            wer  \n",
       "12     0.142857  \n",
       "9532   0.166667  \n",
       "9533   0.222222  \n",
       "12305  0.000000  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.iloc[[12, 9532, 9533,12305]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ec04e1a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6f993e4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "dc80ade9",
   "metadata": {},
   "source": [
    "### wer stats for test set items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "95b1720b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    15801.000000\n",
       "mean         0.181311\n",
       "std          0.194821\n",
       "min          0.000000\n",
       "25%          0.000000\n",
       "50%          0.142857\n",
       "75%          0.285714\n",
       "max          1.200000\n",
       "Name: wer, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgcAAAFFCAYAAAB42U7BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA4c0lEQVR4nO3deVwV9f4/8BcgR+SA5oJLXU0UAk3ZYjMXFDDURAWyxBK/YanhVnlRk7yWuWHghltaUrjcNK+CmKZmSC4ooOaCgKBUYEUKuByOegTm9wc/JkdQz8HDsL2ej4ePh2c+Z2Y+8+Y8zrzOzGdmDARBEEBERET0/xnWdgeIiIiobmE4ICIiIgmGAyIiIpJgOCAiIiIJhgMiIiKSYDggIiIiCYYDoqewc+dOeHp6iq9nzZqFWbNm6XUdJ0+ehI2Njfg6KioKY8aM0es6qloP1U818RmkxofhgOq06OhoeHl5Sabt378fNjY2iI2NlUwfPHgw1qxZAwAYM2YMbGxsKv37+eefAZTvYCumvfjii/D29kZUVBTKysqeqr9hYWEICwvT6r02NjY4efLkE9/n6OiIo0ePPlW/HlZVwKiJ9dRFVX0uHvyXl5dXreWOGTMGUVFReu7tP/Ly8rTq38OfQU9PT+zcubPG+kUNU5Pa7gDR4zg7O2Px4sX466+/0L59ewBAamoq2rdvj5SUFIwYMQIAUFhYiJycHLi4uIjzBgcHIzg4WLK8Fi1aiP93dHREVFQUSktLceHCBcycORMtW7bEW2+9Ve3+mpubV3veqmg0GigUClhYWOh1uVWRaz1yqqjfgx4MQBs3bsSZM2ckO/VWrVrJ1r+aoO/PIDVOPHJAdVr37t2hVCqRmpoqTjt16hTefvttybTTp0/D2NgYdnZ24jRTU1NYWFhI/j24ozA2NoaFhQXat28Pb29vvPzyy0hKSnpsf06ePInBgwfDzs4OEyZMQFFRkaT94UO6X3/9NTw9PdGjRw/069dP3AlVnIoICgqCjY2NOM+YMWPw+eef46OPPoKjoyNWrlz5yMP969evh7u7O1xcXLB8+XJxelW/MB9cxs6dO7Fq1SokJydLfi1XtZ7o6Gj0798fPXv2xKhRo5CWlia2VZxS2bt3Lzw9PeHi4oKwsDBoNJpH1q+0tBQRERF4+eWXYWdnh3HjxiE3NxcAcPHiRXTv3h2FhYWSeUaPHi0eESotLcXy5cvRr18/ODo6YsyYMcjMzBTfW3FEZMOGDejTp0+Vp18e/DyYmpqKnwMLCwuYmZlh/vz5cHd3h7OzMyZOnIg//vhDnPfYsWMYMWIE7Ozs4ObmhvHjxwMo/7snJydj1apVsLGxkZxqelBRURGmTp0KV1dXODg4YNiwYThz5ozYfvz4cfj7+8POzg4+Pj7YunWr2FZxBM3Lyws2NjaPPErx4GdwzJgxuHr1Kj766CPY2NhI6hETEwMvLy/Y29sjICAAKSkpYlvF3zYuLg79+/fHSy+9hNWrV+PevXuYPXs2HB0dMXToUFy8eFGcJy0tDYGBgXBwcICLiwveeust3Lp1q8o+Ut3HIwdUpxkZGcHR0REpKSkYOnQoVCoVLl++jJEjRyIqKgrXr19HmzZtcOrUKdjZ2aFp06bVWs+lS5dw5swZvPTSS498z+3btzF58mQMGzYMo0ePRnJyMpYuXfrIX2rnzp1DVFQUli5dCisrK+Tn54s7wh07dqBXr16IioqCo6MjTExMxPm2bt2KSZMmIS4uDkZGRlUeRr5w4QJatmyJzZs3Izs7G7Nnz4alpSWGDx/+xG0dMmQIsrKyJL+YW7VqhatXr0ret3fvXqxcuRKfffYZunXrhujoaIwfPx4HDx6EqakpAKCgoADx8fFYu3YtCgsLMXXqVLz44osYPXp0lev+6quvsGvXLixatAgdOnTA0qVLERISgri4OHTv3h0dO3bE/v37ERgYCADIz8/HmTNnsGDBAgDAqlWrcOTIESxduhQWFhbYuXMnxo0bh/3790OpVIq1adu2Lb7++msYGBg8sR4Pmjt3LgoKCrBhwwYolUps2LAB7733Hnbt2oWysjJMnToVkydPxsCBA6FSqXDixAkA5Yfyf/31Vzg6OiI4OBhGRkZVLn/FihVQqVSIiYmBqakpMjIyYGxsDAC4cuUKpkyZgtmzZ8PV1VX8u7Zu3Ro+Pj747rvvMHLkSHz33Xfo0KGD+Dd4nKioKAwfPhzBwcEYMmSIuK4dO3Zg8+bN+OSTT9C5c2ccPnwY48ePx969e9GhQwcA5X/bQ4cO4csvv0RaWhpmzJiB06dPY+DAgXj33XexbNkyzJkzB//73/8AAKGhofDw8MCSJUtw9+5dnD59WqfaU93CcEB1nrOzM77//nsAwJkzZ2BrawulUgl7e3ukpqZi0KBBSE1NxcsvvyyZ74svvsDGjRsl03766Se0bNkSQPnpCUdHR5SUlECj0cDY2BhvvvnmI/sRHx8PMzMzzJ49G0ZGRujatStSU1Mlv/we9Oeff8LCwgJ9+vSBkZERnnvuOTg5OQH459B1ixYtKh3Kd3BwwDvvvCO+riocCIKABQsWwNzcHFZWVkhPT8eWLVu0CgcmJiaSX8yP8s0332DMmDEYOnQoAOCTTz7BkSNHEB8fjzfeeANA+WH7BQsWiNszaNAgpKSkPDIcxMTEYOrUqfDw8AAALF68GB4eHjh+/Dj69OmDQYMGYd++fWI4+OGHH2BjYwNLS0vcu3cPGzduxM6dO9G1a1cAwAcffIAffvgBhw8fxquvvgoAMDQ0xPz589GsWbMn1uJBeXl52LdvH5KSkmBmZgYAmDdvHlxcXHDu3Dk8//zzUKlUGDx4sHiKy9bWFkD5oXxjY2PxaNWj/PXXX3B2dhbn69Spk9i2YcMGBAYGIiAgAADQsWNHjB07Ftu3b4ePj49Y41atWml9+ueZZ56BkZERzM3NJfOsXbsWc+fORd++fQGUH2FISEhAfHy8eDREo9Hg008/RcuWLWFlZYV169bBxMQEo0aNAgCMGzcOr7/+Ou7cuYNmzZrhzz//hJeXFzp27AgAsLa21qqPVDcxHFCd5+zsjBUrVqCoqAipqaniDtbJyQmpqanw8PBAeno6pk2bJplv1KhRlQ4rPzjmoEePHoiIiMCdO3fw5Zdfom3btnB1dX1kP3JycvDiiy9KfhXa29s/Mhy8/PLLWLZsGQYOHIi+ffvC09MT/fr1e+Kv2W7duj22HQAsLS0lRyzs7Owkh6D14cqVK+KOAgCaNGmCHj164MqVK+K0Nm3aSM7RW1hY4Lfffqtyebdv38a1a9dgb28vTnvmmWdgaWmJK1euoE+fPhg8eDA2bNiAgoICtG7dGj/88AOGDBkCAPj9999x9+5dvPbaa5Ll3r17VzwiA5TXRtdgAADZ2dm4f/++uMN8ePkODg549dVXMXToUPTt2xf9+vWDj4+PVr/gK4wcORIffPABjh07ht69e2Pw4MGwtLQEUH706tKlS9iyZYv4/pKSEjz77LM6b8vjFBcXIy8vD1OnTpV8FjUaDZ577jnxdZs2bcQgDQCtW7eGlZWV+Lri715UVIRmzZrhrbfewrhx49C7d2/07t0bQ4YMkcxP9QvDAdV59vb2MDY2xqlTp8TxBgDw0ksvITw8HL/88gsEQYCjo6NkvhYtWuD5559/5HJNTEzE9kWLFsHHxwfu7u7o169fle8XBKHSjv1xDzU1NzfH7t27kZSUhCNHjojjCFavXv3Y7dVmx/a4gGFoaFipbyUlJU9cZnVUHKZ+sF9P86BXW1tbdO7cGfv374eXlxd++eUXLFmyBACgVqsBlJ92eXiH/GDoq04wqFi+UqmscmR/69atAQBLly7F2bNnkZiYiC+//BKrV6/G//73P8n6H8fLyws//vgjEhISkJiYiLVr1yIiIgI+Pj5Qq9UYN24c/Pz8JPM0aaLfr+k7d+4AAJYtW4YuXbpI2iqOmABV/20f7EvFZ7DiCp/p06fD19cXhw8fRlxcHFauXIkdO3aIRxKofuGARKrzFAoF7OzskJSUhPPnz4tHDhwcHHD58mUcPnwY3bp1E885V4exsTH+7//+D59//vkjd26WlpZIS0uTXO547ty5J/bdw8MDH3/8MdatW4cff/wRN27cAFD+pV/dSydzcnKgUqnE1+fPnxd/gVb8ort+/brYfunSJcn8TZo0QWlp6WPXYWlpibNnz4qvS0pKcOHChUo7FG2Zm5ujTZs2kmXeuHEDOTk5kmVWnFrYt2+fOA4BALp27QpjY2P8/fffeP755yX/nnnmmWr16UE2NjZQqVQoKSmptPwHd5r29vaYOnUqdu3ahRs3boiXo2pTUwBo27Yt3njjDaxZswYBAQHYtWsXgPJg9Ouvv1Zad8Wv+Yqdta6fmYf71bp1a1hYWODPP/+stK6KEFRdL7zwAsaPH4/t27ejTZs2+PHHH59qeVR7GA6oXnB2dsbOnTvx3HPPiYcqmzVrBhsbG2zfvl1yCWMFtVqNa9euSf7dvXv3kevw9/fH1atXcejQoSrbfX19cfPmTSxcuBBXrlzBtm3bcOTIkUcuLyEhAVu2bEFmZiZyc3Oxd+9etGrVCs2bNwcAPPvsszhx4gQKCgpQXFysSzkAAB9//DEuX76MAwcOYNOmTeJ5fhMTE/To0QNffPEFcnJy8NNPP1U65fDss8/i119/xZUrV1BYWFjlDicoKAibNm3C999/j8uXL+OTTz6BRqMRxyBUR1BQEFauXInExERcunQJs2bNQseOHSXjRYYMGYLU1FRs374dgwcPFqebmZnhrbfewty5c3HgwAHk5ubizJkziIiIwOXLl6vdpwpdu3bFK6+8gmnTpuHo0aPIzc1FcnIyPv30U9y6dQu5ublYtmwZzp49i6tXr+KHH37AnTt3xHEDzz77LM6dO4f8/HzcvHmzynWsXLkSCQkJyM3Nxfnz53H69Gkx1I0bNw6HDh3CypUrcfnyZWRlZWHHjh3473//C6B8p960aVMcP34chYWF4hGAJ3n22Wdx+vRpXLt2Dbdv34aBgQEmTJiA5cuXY+fOnfj9999x/vx5rFu3DsnJydWq3d27dzF//nykpqbi6tWrSExMxB9//IHOnTtXa3lU+3hageoFZ2dnrFu3rtLVBC+99BLOnz8PZ2fnSvNs3Lix0oDE+fPnY+TIkVWuw8zMDH5+fli3bh28vb0rtTdv3hyrVq3CJ598gm3btomXsj3qXL+5uTn27t2LZcuWobS0FN26dcO6devEw/7//ve/ER4ejvXr12P48OFYvHixVrUAysdL2NjYYPTo0SgtLcWbb74pGYy4YMECzJ49W7zsbtKkSZg5c6bY/sorr2Dfvn0ICAiAWq2uMhANHToUf/31F8LDw1FYWIgXX3wR69evf6ojNOPGjcPNmzcxa9YsFBcXw9nZGWvWrBFrAgBWVlawsrLCpUuXJOEAAGbMmIEWLVpg8eLF+Pvvv9G6dWu4ubnp5cgBAERERGDZsmX46KOPUFRUhHbt2qFPnz5o2rQpmjVrhkuXLmHHjh24desWOnbsiAULFoiDC99++23MmDEDXl5eaNu2LX766adKyzcyMkJ4eDiuXr0Kc3NzeHp6YsqUKQDK/6YbN27E0qVL8eWXX8LExET8JQ6UHwGYOXOm+BmcNGmSOO/jTJ48GXPmzEH//v3h5OSETZs2YcyYMVAoFNiwYQP+85//4JlnnoGjoyMGDRpUrboZGhqioKAAH374IQoLC9GuXTtMnjwZAwYMqNbyqPYZCE9zgpCIiIgaHJ5WICIiIgmGAyIiIpJgOCAiIiIJhgMiIiKSYDggIiIiCdnDQXx8PAYNGgQHBwd4e3uLT9ZLT0+Hv78/7O3t4efnJ3n6GwDs2bMHnp6ecHBwwMSJEyVPwxMEAREREXBzc4OrqyvCw8Of6i5tREREjZmslzIeOXIEc+bMwfLly2FnZ4dr164BAFq2bAkfHx+MHTsWo0ePxrfffouvvvoKBw8ehEKhQFZWFl5//XWsX78e3bt3x3/+8x/cv38fK1euBAB8++23iI6OxjfffAMDAwO8/fbbePPNNx/7EJ2qFBUVo6xMP+Vo3doMBQWqJ7+xkWA9/sFaSLEeUqxHOUV8HMzNmqJgQPXuvdAQ6fOzYWhogJYtH33PEllvgrRq1SpMmjQJDg4OAIB27doBAI4ePYqSkhKMHTsWBgYGCAoKQnR0NJKSkuDh4YH4+HjxefEAMG3aNAwePBgqlQpmZmaIjY1FcHCw+KS04OBgbNu2TedwUFYm6C0cVCyP/sF6/IO1kGI9pFgPQLh9G8B91uIhctVDtnBQWlqKtLQ0DBgwAN7e3tBoNPD29saMGTOQnZ0NW1tbycNkbGxskJ2dDQ8PD2RlZYn30wfKH3OqUCiQk5ODnj17IisrS/IkO1tbW2RnZ+vcx9atzZ78Jh1YWJg/+U2NCOvxD9ZCivWQYj0AmJsAYC0eJlc9ZAsH169fx/3793HgwAFs3boVTZo0QUhICNauXQuFQiF5sAkAKJVK8SlsarW6UruZmZmk/cFbula0VfUUvccpKFDpLZVZWJjj2rXbellWQ8B6/IO1kGI9pFiPck1v30VzcxPW4gH6/GwYGho89gexbAMSTUzKU2BQUBDatm2LVq1aITg4GImJiVAqlZInzAHlzxyveCyrqalppXaVSiVpf/DBNRVtugQDIiIiKidbOGjRooU4JqBCxVhIKysrZGZmSq4wyMzMhJWVFQDA2toaGRkZYltubi40Go34NLOH2zMyMsR5iYiISDeyXsro7++PTZs2oaCgADdv3sQ333yD/v37w9XVFUZGRoiJiYFGo8HmzZtRVlaGXr16ASh/VG5CQgJSU1OhVquxYsUKeHl5iacahg8fjujoaOTn5yM/Px/R0dEYMWKEnJtGRETUYMh6tUJISAhu3LiBQYMGQaFQYNCgQXjvvfegUCiwZs0ahIWFITIyEl26dMHq1auhUCgAlB8ZmDdvHkJDQ1FUVAR3d3csXLhQXO6oUaOQl5cHX19fCIKAgIAABAYGyrlpREREDQYf2fwADkisOazHP1gLKdZDivUo1/TbLeUDEl8NqO2u1BkNckAiERER1Q8MB0RERCQh65iDxkRzv/SxN6u4e68Et2/dkbFHRERE2mE4qCEKYyP4To97ZHt85HDwrCIREdVFPK1AREREEgwHREREJMFwQERERBIMB0RERCTBcEBEREQSDAdEREQkwXBAREREEgwHREREJMFwQERERBIMB0RERCTBcEBEREQSDAdEREQkwXBAREREEgwHREREJMFwQERERBIMB0RERCTBcEBEREQSDAdEREQkwXBAREREEgwHREREJMFwQERERBIMB0RERCTBcEBEREQSDAdEREQkwXBAREREEgwHREREJMFwQERERBIMB0RERCTBcEBEREQSDAdEREQkwXBAREREErKGgzFjxqBnz55wdHSEo6MjfHx8xLb09HT4+/vD3t4efn5+SEtLk8y7Z88eeHp6wsHBARMnTkRRUZHYJggCIiIi4ObmBldXV4SHh0MQBNm2i4iIqCGR/cjBf/7zH5w5cwZnzpzB/v37AQAajQYhISEYNmwYUlJS4Ofnh5CQEGg0GgBAVlYW5syZg/DwcBw7dgxKpRJz584Vl7lt2zYcPHgQcXFxiI+PR2JiIrZu3Sr3phERETUIdeK0QnJyMkpKSjB27FgoFAoEBQXB0NAQSUlJAID4+Hh4enrCxcUFSqUS06ZNw6FDh6BSqQAAsbGxCA4ORvv27dGuXTsEBwcjNja2FreIiIio/moi9wojIyMREREBS0tLfPDBB3Bzc0N2djZsbW1hYGAgvs/GxgbZ2dnw8PBAVlYWnJycxLZOnTpBoVAgJycHPXv2RFZWFrp16ya229raIjs7W+e+tW5t9nQbpyMLC3NZ11fbGtv2Pg5rIcV6SLEeAMxNALAWD5OrHrKGg3//+9/o2rUrFAoFvv/+e0ycOBFxcXEoLi6GmZl0x6xUKqFWqwEAarW6UruZmZmkXalUVmoTBEESOJ6koECFsjL9jFXQ5g947dptvayrPrCwMG9U2/s4rIUU6yHFepRrevsumpubsBYP0Odnw9DQ4LE/iGU9rWBvbw8zMzMoFAr4+fnByckJiYmJUCqV4imCCsXFxTA1NQUAmJqaVmpXqVSS9uLi4kptugQDIiIiKlerYw4MDAwgCAKsrKyQmZkpucIgMzMTVlZWAABra2tkZGSIbbm5udBoNLC0tKyyPSMjQ5yXiIiIdCNbOLh16xaOHDmCe/fuoaSkBLt370Zqair69OkDV1dXGBkZISYmBhqNBps3b0ZZWRl69eoFAPD19UVCQgJSU1OhVquxYsUKeHl5iacahg8fjujoaOTn5yM/Px/R0dEYMWKEXJtGRETUoMg25qCkpATLly/HlStXYGRkhC5dumD16tXo0qULAGDNmjUICwtDZGSk2KZQKACUHxmYN28eQkNDUVRUBHd3dyxcuFBc9qhRo5CXlwdfX18IgoCAgAAEBgbKtWlEREQNioHAuwWJ9D0g0Xd63CPb4yOHN6qBNhxk9Q/WQor1kGI9yjX9dkv5gMRXA2q7K3VGgx2QSERERHUfwwERERFJMBwQERGRBMMBERERSTAcEBERkQTDAREREUkwHBAREZEEwwERERFJMBwQERGRBMMBERERSTAcEBERkQTDAREREUkwHBAREZEEwwERERFJMBwQERGRBMMBERERSTAcEBERkQTDAREREUkwHBAREZEEwwERERFJMBwQERGRBMMBERERSTAcEBERkQTDAREREUkwHBAREZEEwwERERFJMBwQERGRBMMBERERSTAcEBERkQTDAREREUkwHBAREZEEwwERERFJMBwQERGRBMMBERERSdRKOCgqKoKbmxsCAwPFaenp6fD394e9vT38/PyQlpYmmWfPnj3w9PSEg4MDJk6ciKKiIrFNEARERETAzc0Nrq6uCA8PhyAIsm0PERFRQ1Ir4WDJkiWwsrISX2s0GoSEhGDYsGFISUmBn58fQkJCoNFoAABZWVmYM2cOwsPDcezYMSiVSsydO1ecf9u2bTh48CDi4uIQHx+PxMREbN26VfbtIiIiaghkDwfJycnIycmBv7+/ZFpJSQnGjh0LhUKBoKAgGBoaIikpCQAQHx8PT09PuLi4QKlUYtq0aTh06BBUKhUAIDY2FsHBwWjfvj3atWuH4OBgxMbGyr1pREREDYKs4UCj0eCzzz7DJ598AgMDA3F6dnY2bG1tJdNsbGyQnZ0NoPzIga2trdjWqVMnKBQK5OTkiO3dunUT221tbcV5iYiISDdN5FzZ+vXr0atXL9ja2uLixYvi9OLiYpiZmUneq1QqoVarAQBqtbpSu5mZmaRdqVRWahMEQRI4nqR1a7Mnv0mPLCzMZV1fbWts2/s4rIUU6yHFegAwNwHAWjxMrnrIFg5+++037Nq1C3FxcZXalEqleIqgQnFxMUxNTQEApqamldpVKpWkvbi4uFKbLsEAAAoKVCgr089ARm3+gNeu3dbLuuoDCwvzRrW9j8NaSLEeUqxHuaa376K5uQlr8QB9fjYMDQ0e+4NYttMKp06dwvXr1+Hj44PevXtjwYIFOH/+PHr37o2uXbsiMzNTcoVBZmamOGjR2toaGRkZYltubi40Gg0sLS2rbM/IyJAMeCQiIiLtyXbkYMiQIejbt6/4eu/evdizZw/WrFmDFi1awMjICDExMQgMDMT27dtRVlaGXr16AQB8fX3xxhtvIDU1Fd27d8eKFSvg5eUlnmoYPnw4oqOj4eHhAQCIjo7G6NGj5do0IiKiBkW2cGBiYgITExPxtbm5OZo0aQILCwsAwJo1axAWFobIyEh06dIFq1evhkKhAFB+ZGDevHkIDQ1FUVER3N3dsXDhQnFZo0aNQl5eHnx9fSEIAgICAiT3UCAiIiLtGQi8W5BI32MOfKdXHl9RIT5yeKM6l8bzqP9gLaRYDynWo1zTb7eUjzl4NaC2u1JnNMgxB0RERFQ/MBwQERGRBMMBERERSTAcEBERkQTDAREREUloHQ4evMkQERERNVxah4PXXnsNw4YNw8aNG/H333/XZJ+IiIioFmkdDn7++WcEBARg7969GDBgAMaNG4e4uDjcuXOnJvtHREREMtM6HLRq1Qpjx47Fjh07EB8fDzs7O6xatQq9e/fGrFmzkJSUVJP9JCIiIplUa0Bily5d4OHhgT59+kCj0SAlJQXvv/8+fHx8kJycrO8+EhERkYx0erbCb7/9ht27dyM+Ph6FhYUYNGgQvv76azg7O0Oj0WD16tUIDQ1FYmJiTfWXiIiIapjW4eD1119HWloa3N3dMWXKFAwcOFDyICWFQoExY8bgiy++qJGOEhERkTy0Dgfe3t6IiopCu3btHvmeNm3a8JJHIiKiek7rMQd+fn6o6gGOf/31F65fv67XThEREVHt0TocTJ8+HceOHas0/cSJE5g+fbpeO0VERES1R+twcOHCBTg7O1ea7uTkhLS0NL12ioiIiGqP1uGgSZMmuHnzZqXphYWFeu0QERER1S6tw0Hfvn3x+eefS8JAYWEhIiIi0Ldv3xrpHBEREclP66sVwsLCMGnSJHh6esLS0hIGBga4cuUKbG1tERYWVpN9JCIiIhlpHQ5atWqF//73vzhx4gSys7MhCAKsra3h7u5ek/0jIiIimel0h0QAcHd3ZyAgIiJqwHQKB0eOHEFKSgoKCgpQVlYmaVu0aJFeO0ZERES1Q+sBiZGRkXjvvfeQlpaG0tLSmuwTERER1SKtjxzs2LEDy5Ytw8CBA2uyP0RERFTLtD5yYGRkhK5du9ZkX4iIiKgO0DocjB8/Hl999RVKSkpqsj9ERERUy7Q+rXDw4EFcvHgRCQkJ6Ny5M5o0kc4aExOj984RERGR/LQOB25ubnBzc6vJvhAREVEdoHU4mDx5ck32g4iIiOoIrcccAMCdO3ewd+9erF+/Hrdu3QIA5OTk8OFLREREDYjWRw4uXbqEcePGwdTUFHl5eRg8eDCaN2+OnTt34tq1a1i8eHFN9pOIiIhkovWRg/nz5yMgIAD79++HQqEQp3t6euLkyZM10jkiIiKSn9bh4MKFCwgICKg03cLCAtevX9drp4iIiKj2aB0OzM3Nce3atUrT09PT0a5dO712ioiIiGqP1uHAz88PixYtQk5ODgwMDKBWq3HkyBEsWrQII0eOrMk+EhERkYy0DgdTp05Fnz594OfnB7VajREjRmDSpEnw8fHBhAkTtFrGkiVL4OHhAScnJwwYMABr164V29LT0+Hv7w97e3v4+fkhLS1NMu+ePXvg6ekJBwcHTJw4EUVFRWKbIAiIiIiAm5sbXF1dER4eDkEQtN00IiIieoDW4cDQ0BDTpk1DcnIy9uzZg23btuH48eOYOXOm1it77bXXsG/fPpw+fRrffvst9uzZg/3790Oj0SAkJATDhg1DSkoK/Pz8EBISAo1GAwDIysrCnDlzEB4ejmPHjkGpVGLu3Lnicrdt24aDBw8iLi4O8fHxSExMxNatW3UoAxEREVXQ6T4HAKBQKGBlZQU7OzuYmZnpNG+XLl1gamoqvjYwMMDvv/+O5ORklJSUYOzYsVAoFAgKCoKhoSGSkpIAAPHx8fD09ISLiwuUSiWmTZuGQ4cOQaVSAQBiY2MRHByM9u3bo127dggODkZsbKyum0ZERETQ4T4HY8aMgYGBwSPbtX22wvr167F27Vqo1Wp06tQJw4YNw759+2BraytZvo2NDbKzs+Hh4YGsrCw4OTmJbZ06dYJCoUBOTg569uyJrKwsdOvWTWy3tbVFdna2tpsmat1at7DztCwszGVdX21rbNv7OKyFFOshxXoAMDcBwFo8TK566PRshQfdv38fly5dQmpqKkaPHq31CsePH493330X6enpOHjwIJRKJYqLiysdhVAqlVCr1QAAtVpdqd3MzEzSrlQqK7UJgvDYQPOwggIVysr0M1ZBmz/gtWu39bKu+sDCwrxRbe/jsBZSrIcU61Gu6e27aG5uwlo8QJ+fDUNDg8f+IH7qZyvExMTg0qVLOnXKwMAA3bt3x9GjRxEVFYUOHTqIpwgqFBcXi6cgTE1NK7WrVCpJe3FxcaU2XYIBERERldN5zMHD+vfvj71791Zr3pKSEvz++++wsrJCZmam5AqDzMxMWFlZAQCsra2RkZEhtuXm5kKj0cDS0rLK9oyMDHFeIiIi0s1ThQNBEBAXF4dWrVo98b1lZWXYtm0bbt26hbKyMpw9exZbt25Fr1694OrqCiMjI8TExECj0WDz5s0oKytDr169AAC+vr5ISEhAamoq1Go1VqxYAS8vL/FUw/DhwxEdHY38/Hzk5+cjOjoaI0aMeJpNIyIiarS0Pq3g6ekpOUwvCIJ4r4GFCxdqtYwDBw4gMjIS9+/fR9u2bfHWW2+JAx3XrFmDsLAwREZGokuXLli9erX4DAdra2vMmzcPoaGhKCoqgru7u2Sdo0aNQl5eHnx9fSEIAgICAhAYGKjtphEREdEDDAQt7xa0a9cu6YwGBmjVqhV69uyJli1b1kjn5KbvAYm+0+Me2R4fObxRDbThIKt/sBZSrIcU61Gu6bdbygckvlr5mT6NVZ0ckOjn56eXDhEREVHdpnU4SElJ0XqhLi4u1eoMERER1b5q3QSp4kzEw68rpqWnp+uzj0RERCQjrcPBhg0bsGLFCnzwwQewt7cHAJw9exbLly/HlClT0K9fvxrrJBEREclH63CwYMEChIeHi8EAAHr37g2lUokZM2bgwIEDNdJBIiIikpfW9zn4888/q7zjoKGhIfLz8/XaKSIiIqo9WoeDfv36Yfbs2Thx4gSKi4uhVqtx4sQJhIWF8ZQCERFRA6L1aYVFixZhwYIFeOedd1BaWgoAMDIywtChQzF79uwa6yARERHJS+twYGZmhkWLFiEsLAx5eXkQBAEdO3as9LREIiIiqt90frbCvXv3cPfuXXTt2pXBgIiIqAHSOhzcvHkTEydORO/evREYGCgOQpw7dy6WLVtWYx0kIiIieWkdDubPnw8ASExMhImJiTh94MCBOHjwoP57RkRERLVC6zEHR44cwebNm9GuXTvJ9Oeffx5//PGH3jtGREREtUPrIwcVVyg87O+//4apqaneOkRERES1S6f7HHz11VeS5yjcuHEDy5Ytg6enZ410joiIiOSndTiouISxf//+uHfvHkJCQjBgwABoNBrMmDGjJvtIREREMtLpPgcxMTE4deoUMjMzoVarYWtriz59+lR5W2UiIiKqn7QKB/fv34ejoyN2794NZ2dnODs713S/iIiIqJZodVrB2NgYzz//PFQqVU33h4iIiGqZ1mMOPvroIyxZsgS//PIL7t27V5N9IiIiolqk9ZiDd999FwAwatSoKscYpKen669XREREVGueGA7effddLF26FDExMQAAtVoNAwMDNGvWrMY7R0RERPJ7Yjg4evQoNBoNXF1dAQBOTk6Ii4tDx44da7xzREREJL8njjl48KZHVb0mIiKihkXnRzYTERFRw6bVgMSvvvpKfH7C/fv3ERMTgxYtWkjeM3nyZP33joiIiGT3xHDg4uKC8+fPi68dHR2RkZEheQ/vkEhERNRwPDEcbNq0SY5+EBERUR3BMQdEREQkwXBAREREEgwHREREJMFwQERERBIMB0RERCTBcEBEREQSDAdEREQkIVs40Gg0mD17NgYMGABHR0cMGzYMP/74o9ienp4Of39/2Nvbw8/PD2lpaZL59+zZA09PTzg4OGDixIkoKioS2wRBQEREBNzc3ODq6orw8HA+A4KIiKiaZAsHJSUl6NChAzZt2oRTp05h+vTpCA0NxW+//QaNRoOQkBAMGzYMKSkp8PPzQ0hICDQaDQAgKysLc+bMQXh4OI4dOwalUom5c+eKy962bRsOHjyIuLg4xMfHIzExEVu3bpVr04iIiBoU2cKBqakppkyZgn/9618wNDSEh4cHOnfujLS0NCQnJ6OkpARjx46FQqFAUFAQDA0NkZSUBACIj4+Hp6cnXFxcoFQqMW3aNBw6dAgqlQoAEBsbi+DgYLRv3x7t2rVDcHAwYmNj5do0IiKiBkWrBy/VhMLCQly5cgVWVlY4fvw4bG1tJc9osLGxQXZ2Njw8PJCVlQUnJyexrVOnTlAoFMjJyUHPnj2RlZWFbt26ie22trbIzs7WuU+tW5s93UbpyMLCXNb11bbGtr2Pw1pIsR5SrAcAcxMArMXD5KpHrYSDkpISzJw5E4MHD8YLL7yAgwcPwsxMumNWKpVQq9UAALVaXandzMxM0q5UKiu1CYKg00OhCgpUKCvTz1gFbf6A167d1su66gMLC/NGtb2Pw1pIsR5SrEe5prfvorm5CWvxAH1+NgwNDR77g1j2qxXKysowc+ZMlJWVYd68eQDKg0DFKYIKxcXF4mOiTU1NK7WrVCpJe3FxcaU2Pi2SiIhId7KGA0EQEBYWhvz8fKxatQoKhQIAYGVlhczMTMkVBpmZmbCysgIAWFtbSx4TnZubC41GA0tLyyrbMzIyxHmJiIhIN7KGg7lz5+Ly5ctYt24dmjVrJk53dXWFkZERYmJioNFosHnzZpSVlaFXr14AAF9fXyQkJCA1NRVqtRorVqyAl5eXeKph+PDhiI6ORn5+PvLz8xEdHY0RI0bIuWlEREQNhmxjDq5evYpt27ZBoVCgb9++4vQJEyZg4sSJWLNmDcLCwhAZGYkuXbpg9erV4pEFa2trzJs3D6GhoSgqKoK7uzsWLlwoLmPUqFHIy8uDr68vBEFAQEAAAgMD5do0IiKiBsVA4N2CRPoekOg7Pe6R7fGRwxvVQBsOsvoHayHFekixHuWafrulfEDiqwG13ZU6o0EPSCQiIqK6jeGAiIiIJBgOiIiISILhgIiIiCQYDoiIiEiC4YCIiIgkGA6IiIhIguGAiIiIJBgOiIiISILhgIiIiCQYDoiIiEiC4YCIiIgkGA6IiIhIguGAiIiIJBgOiIiISILhgIiIiCQYDoiIiEiC4YCIiIgkGA6IiIhIguGAiIiIJBgOiIiISILhgIiIiCQYDoiIiEiC4YCIiIgkGA6IiIhIguGAiIiIJBgOiIiISILhgIiIiCQYDoiIiEiC4YCIiIgkGA6IiIhIguGAiIiIJBgOiIiISILhgIiIiCRkDQebN2+Gv78/evTogVmzZkna0tPT4e/vD3t7e/j5+SEtLU3SvmfPHnh6esLBwQETJ05EUVGR2CYIAiIiIuDm5gZXV1eEh4dDEARZtomIiKihkTUctG3bFiEhIQgICJBM12g0CAkJwbBhw5CSkgI/Pz+EhIRAo9EAALKysjBnzhyEh4fj2LFjUCqVmDt3rjj/tm3bcPDgQcTFxSE+Ph6JiYnYunWrnJtGRETUYMgaDl555RV4e3vjmWeekUxPTk5GSUkJxo4dC4VCgaCgIBgaGiIpKQkAEB8fD09PT7i4uECpVGLatGk4dOgQVCoVACA2NhbBwcFo37492rVrh+DgYMTGxsq5afWSefNmsLAwf+Q/8+bNaruLRERUC5rUdgcAIDs7G7a2tjAwMBCn2djYIDs7Gx4eHsjKyoKTk5PY1qlTJygUCuTk5KBnz57IyspCt27dxHZbW1tkZ2fLug31kUnTJvCdHvfI9vjI4bgtY3+IiKhuqBPhoLi4GGZmZpJpSqUSarUaAKBWqyu1m5mZSdqVSmWlNkEQJIHjSVq3Nnvym/TIwsJc1vVVhz77WB+2Vy6shRTrIcV6ADA3AcBaPEyuetSJcKBUKsVTBBWKi4thamoKADA1Na3UrlKpJO3FxcWV2nQJBgBQUKBCWZl+BjJq8we8dq12f5fL2UcLC/Na3966grWQYj2kWI9yTW/fRXNzE9biAfr8bBgaGjz2B3GduJTRysoKmZmZkisMMjMzYWVlBQCwtrZGRkaG2JabmwuNRgNLS8sq2zMyMsR5iYiISDeyhoOSkhLcu3cPZWVlKC0txb1793D//n24urrCyMgIMTEx0Gg02Lx5M8rKytCrVy8AgK+vLxISEpCamgq1Wo0VK1bAy8tLPNUwfPhwREdHIz8/H/n5+YiOjsaIESPk3LQGSXO/lAMWiYgaIVlPK6xduxarVq0SX+/evRt+fn5YvHgx1qxZg7CwMERGRqJLly5YvXo1FAoFgPIjA/PmzUNoaCiKiorg7u6OhQsXissZNWoU8vLy4OvrC0EQEBAQgMDAQDk3rUFSGBtxwCIRUSMkaziYMmUKpkyZUmVbt27dsHPnzkfOO3ToUAwdOrTKNgMDA4SGhiI0NFQv/ST9MG/eDCZN//mIPTzG4e69Ety+dUfubhER0RPUiQGJ1DDxUkkiovqpTgxIJCIiorqD4YCIiIgkGA6IiIhIguGAiIiIJBgOiIiISIJXK1Cd9fClkA/jpZBERDWD4YDqLF4KSURUO3hagYiIiCR45IDqrYpnPzyKNqcdeOqCiKgyhoMG6kk7vYZAH89+4KkLIqLKGvbeoxF70k4PKN/xERERPYxjDoiIiEiCRw7qqcZw2oCIiGoH9y71lDbnyomIiKqD4YCoAePVGERUHQwHRA0Yr8YgoupgOCCqQY/65V5xfwb+cieiuojhgGrNk25i1BDwlzsR1UcMB1RrtLmJERERyY/3OSAiIiIJhgMiIiKSYDggIiIiCY45IHoKvFMlETVE/FYjegpPe6dKfTx2mohI3xgOqMGqD5dK6uOx00RE+sZwQA3Wk3a8AC+XJCKqCsMBVVt9+GVORES6YzigauNNjIiIGiZeykhEREQSPHJARLWKV2wQ1T0MB0R1WGPYcfKKDaK6h+GAqA6rDzvOJ90IqiEEGKLGhuGAiJ4KH0tN1PAwHNRRvC0vkXZ45IJI/xrM3ufGjRsICwvD8ePH0aJFC7z//vsYMWJEbXer2p72trykH7yXQ+0HVW3+BjxyQaRfDSYczJs3D8bGxjh69CjS09Mxfvx4dOvWDTY2NrXdNarHeC+H2g+qNf030MeRh6ddxsPzPxyGGsPRj0o1NDcB8E8tGkMN6pIGEQ7UajUOHDiA+Ph4KJVKODs7w9vbG7t370ZoaKjWyzE0NNBrv9q2bPZU63vS/E/bLsc66np7XejD07Rr86v6aT+H2iyjrn/WH7d+k6ZNMG7+gUe2f/XxKyh+Qv+ftIy1M72e+Hd6mvnv3SuBSnX3scuv6x6uYe+sXwEAx/LKp2nzd2gM9LWfetJyDARBEPSyplp08eJFBAYG4uzZs+K0jRs34uTJk/jiiy9qsWdERET1T4O4Q6JarYaZmZlkmpmZGdRqdS31iIiIqP5qEOHA1NQUKpVKMk2lUsHU1LSWekRERFR/NYhw0LlzZ5SWluLXX38Vp2VkZMDKyqr2OkVERFRPNYhwYGpqioEDB2LlypVQq9U4deoUDh06BF9f39ruGhERUb3TIAYkAv/c5+DYsWNo0aIFPvjgg3p9nwMiIqLa0mDCAREREelHgzitQERERPrDcEBEREQSDAdEREQkwXBAREREEgwH1XTjxg1MmjQJjo6O6N+/P2JjYx/53j179sDT0xMODg6YOHEiioqK5OuoTLStx+HDhxEYGAhnZ2f07t0bH330EW7duiVvZ2uYLp+NClFRUbCxscHx48drvoMy06UeBQUF+PDDD+Hs7AwXFxdMnz5dvo7KRJd6bNmyBd7e3nBycoKvry8OHz4sWz/lsHnzZvj7+6NHjx6YNWvWY9/bGL5Hta2HHN+jDAfV9OBTICMiIjBv3jxkZmZWel9WVhbmzJmD8PBwHDt2DEqlEnPnzq2FHtcsbetx+/ZtvPfeezhy5Aj27duHwsJCLFq0qBZ6XHO0rUWFnJwc7N+/HxYWFjL2Uj661GPSpEmwsLBAQkICjh8/jnHjxsnc25qnbT3OnTuHzz//HEuXLsWpU6cwZcoUTJs2rUHtFNu2bYuQkBAEBAQ89n2N5XtU23rI8j0qkM6Ki4uFF198Ubhy5Yo4LTQ0VFiyZEml90ZGRgoffvih+Pq3334TunfvLty+fVuWvspBl3o87McffxQGDRpUk92TVXVqERQUJPz888/CgAEDhGPHjsnRTdnoUo+KGpSUlMjZRVnpUo/vv/9eCAgIkExzdnYWzp49W+P9lNvSpUuFmTNnPrK9MXyPPuhJ9XhYTXyP8shBNfz6668wMjKCpaWlOM3W1hbZ2dmV3puVlQVbW1vxdadOnaBQKJCTkyNLX+WgSz0edurUKVhbW9dk92Slay1iY2PRokUL9O3bV64uykqXepw9exaWlpaYMWMG3NzcEBAQgOTkZDm7W+N0qUe/fv1QWlqK06dPo7S0FHv37oVSqcQLL7wgZ5frhMbwPfo0auJ7tIlel9ZI6PIUyMbwxMjqbuPJkyfx3XffYevWrTXZPVnpUosbN24gKioKW7Zskat7stOlHvn5+Th69Cjmz5+PxYsX48CBAwgJCcGBAwfQqlUrubpco3Sph6mpKby9vREUFISysjI0bdoUq1atgomJiVzdrTMaw/doddXU9yiPHFSDLk+BbAxPjKzONp47dw7vv/8+li1b1qCOHOhSi88//xyjR49G+/bt5eqe7HSpR9OmTfGvf/0LI0eOhLGxMV599VV06NABp0+flqu7NU6Xemzfvh27du3C7t27ceHCBURFReHDDz/E1atX5epundEYvkeroya/RxkOqkGXp0BaW1sjIyNDfJ2bmwuNRiM5rFjf6fpUzIsXL2LChAn47LPP0KdPH5l6KQ9dapGUlIQvv/wSvXv3Ru/evfHnn3/i/fffx9q1a2Xscc3SpR42NjYy9qx26FKPS5cuYcCAAejSpQsMDQ3Rp08fPPfcc/jll1/k63Ad0Ri+R3VV09+jDAfVoMtTIH19fZGQkIDU1FSo1WqsWLECXl5elQ6R1We61OPSpUt455138PHHH8Pb27sWeluzdKnFjh07sHv3bsTGxiI2NhZt27bFp59+iqCgoFroec3QpR4DBw7ErVu3sGvXLpSWluKHH35Afn4+nJycaqHnNUOXetjZ2SExMRG///47BEFAUlISLl++3KCOtJWUlODevXsoKytDaWkp7t27h/v371d6X2P4HgW0r4cs36N6Hd7YiBQVFQkhISGCvb290K9fP2HXrl1im4ODg5CSkiK+jo+PF/r37y/Y29sLEyZMEAoKCmqhxzVL23rMmjVLsLGxERwcHCT/GhJdPhsPaohXKwiCbvVISUkRhg4dKjg4OAh+fn6PrFV9pm09ysrKhJUrVwr9+/cXHBwcBB8fH2Hnzp211OuasXLlSuGFF16Q/KsYpd8Yv0e1rYcc36N8KiMRERFJ8LQCERERSTAcEBERkQTDAREREUkwHBAREZEEwwERERFJMBwQERGRBMMBERERSTAcEBERkQTDAREREUn8PwmrxQzerVVEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df_results['wer'].describe())\n",
    "\n",
    "df_results['wer'].plot.hist(bins=50, figsize=(8, 5));\n",
    "plt.title('WER distribution over Test set items');\n",
    "plt.axvline(1, color='red', alpha=0.5);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f96087c7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f43696a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0997463c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed13289e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}