week10/seminar07.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fast Speech I"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "1XEo6WrJcXlE"
   },
   "outputs": [],
   "source": [
    "from dataclasses import dataclass\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch import nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import time\n",
    "import os\n",
    "\n",
    "from tqdm import tqdm\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "\n",
    "import seaborn as sns \n",
    "sns.set()\n",
    "\n",
    "import sys\n",
    "sys.path.append(\".\")\n",
    "from text.symbols import symbols\n",
    "from text import text_to_sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mels and LJSpeech with text construction you can download [here](https://disk.yandex.ru/d/FwBnIRrsAYnbgQ), it is needed only for dataset construction.\n",
    "\n",
    "The original code and the repository can be found [here](https://github.com/xcmyz/FastSpeech)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class MelSpectrogramConfig:\n",
    "    num_mels = 80\n",
    "\n",
    "@dataclass\n",
    "class FastSpeechConfig:\n",
    "    vocab_size = 300\n",
    "    max_seq_len = 3000\n",
    "\n",
    "    encoder_dim = 256\n",
    "    encoder_n_layer = 4\n",
    "    encoder_head = 2\n",
    "    encoder_conv1d_filter_size = 1024\n",
    "\n",
    "    decoder_dim = 256\n",
    "    decoder_n_layer = 4\n",
    "    decoder_head = 2\n",
    "    decoder_conv1d_filter_size = 1024\n",
    "\n",
    "    fft_conv1d_kernel = (9, 1)\n",
    "    fft_conv1d_padding = (4, 0)\n",
    "\n",
    "    duration_predictor_filter_size = 256\n",
    "    duration_predictor_kernel_size = 3\n",
    "    dropout = 0.1\n",
    "    \n",
    "    PAD = 0\n",
    "    UNK = 1\n",
    "    BOS = 2\n",
    "    EOS = 3\n",
    "\n",
    "    PAD_WORD = '<blank>'\n",
    "    UNK_WORD = '<unk>'\n",
    "    BOS_WORD = '<s>'\n",
    "    EOS_WORD = '</s>'\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class TrainConfig:\n",
    "    checkpoint_path = \"./model_new\"\n",
    "    logger_path = \"./logger\"\n",
    "    mel_ground_truth = \"./mels\"\n",
    "    alignment_path = \"./FastSpeech/alignments\"\n",
    "    data_path = './data/train.txt'\n",
    "    \n",
    "    wandb_project = 'fastspeech_example' #not needed\n",
    "    \n",
    "    text_cleaners = ['english_cleaners']\n",
    "\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    device = \"cpu\"#'cuda:0'\n",
    "\n",
    "    batch_size = 16\n",
    "    epochs = 2000\n",
    "    n_warm_up_step = 4000\n",
    "\n",
    "    learning_rate = 1e-3\n",
    "    weight_decay = 1e-6\n",
    "    grad_clip_thresh = 1.0\n",
    "    decay_step = [500000, 1000000, 2000000]\n",
    "\n",
    "    save_step = 3000\n",
    "    log_step = 5\n",
    "    clear_Time = 20\n",
    "\n",
    "    batch_expand_size = 32\n",
    "    \n",
    "\n",
    "mel_config = MelSpectrogramConfig()\n",
    "model_config = FastSpeechConfig()\n",
    "train_config = TrainConfig()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def pad_1D(inputs, PAD=0):\n",
    "\n",
    "    def pad_data(x, length, PAD):\n",
    "        x_padded = np.pad(x, (0, length - x.shape[0]),\n",
    "                          mode='constant',\n",
    "                          constant_values=PAD)\n",
    "        return x_padded\n",
    "\n",
    "    max_len = max((len(x) for x in inputs))\n",
    "    padded = np.stack([pad_data(x, max_len, PAD) for x in inputs])\n",
    "\n",
    "    return padded\n",
    "\n",
    "\n",
    "def pad_1D_tensor(inputs, PAD=0):\n",
    "\n",
    "    def pad_data(x, length, PAD):\n",
    "        x_padded = F.pad(x, (0, length - x.shape[0]))\n",
    "        return x_padded\n",
    "\n",
    "    max_len = max((len(x) for x in inputs))\n",
    "    padded = torch.stack([pad_data(x, max_len, PAD) for x in inputs])\n",
    "\n",
    "    return padded\n",
    "\n",
    "\n",
    "def pad_2D(inputs, maxlen=None):\n",
    "\n",
    "    def pad(x, max_len):\n",
    "        PAD = 0\n",
    "        if np.shape(x)[0] > max_len:\n",
    "            raise ValueError(\"not max_len\")\n",
    "\n",
    "        s = np.shape(x)[1]\n",
    "        x_padded = np.pad(x, (0, max_len - np.shape(x)[0]),\n",
    "                          mode='constant',\n",
    "                          constant_values=PAD)\n",
    "        return x_padded[:, :s]\n",
    "\n",
    "    if maxlen:\n",
    "        output = np.stack([pad(x, maxlen) for x in inputs])\n",
    "    else:\n",
    "        max_len = max(np.shape(x)[0] for x in inputs)\n",
    "        output = np.stack([pad(x, max_len) for x in inputs])\n",
    "\n",
    "    return output\n",
    "\n",
    "\n",
    "def pad_2D_tensor(inputs, maxlen=None):\n",
    "\n",
    "    def pad(x, max_len):\n",
    "        if x.size(0) > max_len:\n",
    "            raise ValueError(\"not max_len\")\n",
    "\n",
    "        s = x.size(1)\n",
    "        x_padded = F.pad(x, (0, 0, 0, max_len-x.size(0)))\n",
    "        return x_padded[:, :s]\n",
    "\n",
    "    if maxlen:\n",
    "        output = torch.stack([pad(x, maxlen) for x in inputs])\n",
    "    else:\n",
    "        max_len = max(x.size(0) for x in inputs)\n",
    "        output = torch.stack([pad(x, max_len) for x in inputs])\n",
    "\n",
    "    return output\n",
    "\n",
    "\n",
    "def process_text(train_text_path):\n",
    "    with open(train_text_path, \"r\", encoding=\"utf-8\") as f:\n",
    "        txt = []\n",
    "        for line in f.readlines():\n",
    "            txt.append(line)\n",
    "\n",
    "        return txt\n",
    "\n",
    "\n",
    "def get_data_to_buffer(train_config):\n",
    "    buffer = list()\n",
    "    text = process_text(train_config.data_path)\n",
    "\n",
    "    start = time.perf_counter()\n",
    "    for i in tqdm(range(len(text))):\n",
    "\n",
    "        mel_gt_name = os.path.join(\n",
    "            train_config.mel_ground_truth, \"ljspeech-mel-%05d.npy\" % (i+1))\n",
    "        mel_gt_target = np.load(mel_gt_name)\n",
    "        duration = np.load(os.path.join(\n",
    "            train_config.alignment_path, str(i)+\".npy\"))\n",
    "        character = text[i][0:len(text[i])-1]\n",
    "        character = np.array(\n",
    "            text_to_sequence(character, train_config.text_cleaners))\n",
    "\n",
    "        character = torch.from_numpy(character)\n",
    "        duration = torch.from_numpy(duration)\n",
    "        mel_gt_target = torch.from_numpy(mel_gt_target)\n",
    "\n",
    "        buffer.append({\"text\": character, \"duration\": duration,\n",
    "                       \"mel_target\": mel_gt_target})\n",
    "\n",
    "    end = time.perf_counter()\n",
    "    print(\"cost {:.2f}s to load all data into buffer.\".format(end-start))\n",
    "\n",
    "    return buffer\n",
    "\n",
    "\n",
    "class BufferDataset(Dataset):\n",
    "    def __init__(self, buffer):\n",
    "        self.buffer = buffer\n",
    "        self.length_dataset = len(self.buffer)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.length_dataset\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.buffer[idx]\n",
    "\n",
    "\n",
    "def reprocess_tensor(batch, cut_list):\n",
    "    texts = [batch[ind][\"text\"] for ind in cut_list]\n",
    "    mel_targets = [batch[ind][\"mel_target\"] for ind in cut_list]\n",
    "    durations = [batch[ind][\"duration\"] for ind in cut_list]\n",
    "\n",
    "    length_text = np.array([])\n",
    "    for text in texts:\n",
    "        length_text = np.append(length_text, text.size(0))\n",
    "\n",
    "    src_pos = list()\n",
    "    max_len = int(max(length_text))\n",
    "    for length_src_row in length_text:\n",
    "        src_pos.append(np.pad([i+1 for i in range(int(length_src_row))],\n",
    "                              (0, max_len-int(length_src_row)), 'constant'))\n",
    "    src_pos = torch.from_numpy(np.array(src_pos))\n",
    "\n",
    "    length_mel = np.array(list())\n",
    "    for mel in mel_targets:\n",
    "        length_mel = np.append(length_mel, mel.size(0))\n",
    "\n",
    "    mel_pos = list()\n",
    "    max_mel_len = int(max(length_mel))\n",
    "    for length_mel_row in length_mel:\n",
    "        mel_pos.append(np.pad([i+1 for i in range(int(length_mel_row))],\n",
    "                              (0, max_mel_len-int(length_mel_row)), 'constant'))\n",
    "    mel_pos = torch.from_numpy(np.array(mel_pos))\n",
    "\n",
    "    texts = pad_1D_tensor(texts)\n",
    "    durations = pad_1D_tensor(durations)\n",
    "    mel_targets = pad_2D_tensor(mel_targets)\n",
    "\n",
    "    out = {\"text\": texts,\n",
    "           \"mel_target\": mel_targets,\n",
    "           \"duration\": durations,\n",
    "           \"mel_pos\": mel_pos,\n",
    "           \"src_pos\": src_pos,\n",
    "           \"mel_max_len\": max_mel_len}\n",
    "\n",
    "    return out\n",
    "\n",
    "\n",
    "def collate_fn_tensor(batch):\n",
    "    len_arr = np.array([d[\"text\"].size(0) for d in batch])\n",
    "    index_arr = np.argsort(-len_arr)\n",
    "    batchsize = len(batch)\n",
    "    real_batchsize = batchsize // train_config.batch_expand_size\n",
    "\n",
    "    cut_list = list()\n",
    "    for i in range(train_config.batch_expand_size):\n",
    "        cut_list.append(index_arr[i*real_batchsize:(i+1)*real_batchsize])\n",
    "\n",
    "    output = list()\n",
    "    for i in range(train_config.batch_expand_size):\n",
    "        output.append(reprocess_tensor(batch, cut_list[i]))\n",
    "\n",
    "    return output\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "buffer = get_data_to_buffer(train_config) # make sure to have dataset at the locations (irrelevant for further discussion)\n",
    "\n",
    "dataset = BufferDataset(buffer)\n",
    "\n",
    "training_loader = DataLoader(\n",
    "    dataset,\n",
    "    batch_size=train_config.batch_expand_size * train_config.batch_size,\n",
    "    shuffle=True,\n",
    "    collate_fn=collate_fn_tensor,\n",
    "    drop_last=True,\n",
    "    num_workers=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = next(iter(training_loader))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([16, 182])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch['text'].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformer Block"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scaled Dot Product Attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we have computer q,k,v matricies and we should calc attention score and calc weighted value vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"https://img-blog.csdnimg.cn/20190325121034288.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ScaledDotProductAttention(nn.Module):\n",
    "    ''' Scaled Dot-Product Attention '''\n",
    "\n",
    "    def __init__(self, temperature, attn_dropout=0.1):\n",
    "        super().__init__()\n",
    "        self.temperature = temperature\n",
    "        self.dropout = nn.Dropout(attn_dropout)\n",
    "        self.softmax = nn.Softmax(dim=2)\n",
    "\n",
    "    def forward(self, q, k, v, mask=None):\n",
    "        # q, k, v: [ (batch_size * n_heads) x seq_len x hidden_size ]\n",
    "        # attn: [ (batch_size * n_heads) x seq_len x seq_len ]\n",
    "        attn = self.softmax(torch.einsum(\"nik,njk->nij\",q,k)/self.temperature)\n",
    "        attn = self.dropout(attn)#DropConnect\n",
    "        \n",
    "        # output: [ (batch_size * n_heads) x seq_len x hidden_size ]\n",
    "        out = torch.einsum(\"nij,njk->nik\", attn, v)\n",
    "        return out, attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dot_product_attention = ScaledDotProductAttention(1)\n",
    "\n",
    "q = torch.randn(4 * 4, 8, 4)\n",
    "k = torch.randn(4 * 4, 8, 4)\n",
    "v = torch.randn(4 * 4, 8, 4)\n",
    "\n",
    "output, attn = dot_product_attention(q, k, v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's visualize!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAB2gAAAFjCAYAAAD4l1tVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZi0lEQVR4nO3dfZyVdZ34/zd3M0MCIn6DQfGGxYSwGzBHo5Ukad2+G20s2293LVptAzWzyRvQKL6GaEECodQi2RdsS1nbbxppWbHRvY9EtLISodZcFlIYExURmAHO+f3hMus0MDPOmZnrc871fPY4j4dc5xrOe0x5OdfnuulVLBaLAQAAAAAAAEC36531AAAAAAAAAAB5YYEWAAAAAAAAoIdYoAUAAAAAAADoIRZoAQAAAAAAAHqIBVoAAAAAAACAHmKBFgAAAAAAAKCHWKAFAAAAAAAA6CEWaAEAAAAAAAB6iAVaAAAAAAAAgB7SN+sBDqfuuLdmPUK7fvHHx7MeoU29e/XKeoQ2/Xrk67MeoV3/undI1iO0acGOH2c9QrsKhULWI5S9/U1/6Nrf74+/7/TX9vtff9aFk1AuPnHye7MeoV1Ltv806xHadKBwMOsR2vWWV4/JeoQ2feFV1VmP0K7Xb3kk6xHKWnXfflmP0K4X9/xnl/5+pTQ5Qpfz6nXD3pz1CG36x5pTsh6hTT8pPpf1CO360TMbsx6hTbWvSvvn5IiI0f1rsx6hTduans16hHZt35f2jE8917X/nmgyndGv6visR2hTMesBOiD149efrD0n6xHadd32H2U9QpsKxbT/Sdz1+b/LeoR2Db/ynqxHaNPzu7t+nSxPx6+TXKAFoJuUwUIRAOSCJgNAGjQZANKRoy5boAXIk6KrmgEgCZoMAGnQZABIR466bIEWIE/cdhoA0qDJAJAGTQaAdOSoy72zHgAAAAAAAAAgL1xBC5AjxRzdIgIAUqbJAJAGTQaAdOSpyxZoAfIkR7eIAICkaTIApEGTASAdOeqyBVqAPMnRGUgAkDRNBoA0aDIApCNHXbZAC5AnhYNZTwAARGgyAKRCkwEgHTnqcu+sBwCgBxULnX8BAF2nlCZ3UZe/8IUvxPvf//4293n22Wfjqquuirq6ujjzzDPjuuuui71793bJ5wNAEhJoMgDw33LUZFfQAgAA5Mwdd9wRN910U5xxxhlt7ldfXx979+6NL33pS7Fr1674xCc+EXv27InPfOYzPTQpAAAAVB4LtAB5kqOHrANA0jJq8o4dO+KTn/xkrF+/Pk4++eQ29/3FL34RDz74YNx3330xatSoiIiYP39+zJgxI6688soYNmxYD0wMAN3Mz8kAkI4cddktjgFypFgsdPoFAHSdUppcSpcfffTR6NevX9xzzz3xxje+sc19H3rooXj1q1/dvDgbEXHmmWdGr1694uGHH+70DACQkqyaDAC0lqcmu4IWIE9ydAYSACStxCZPnjy5zffXrVt32O3nnntunHvuuR36jB07dsTw4cNbbKuqqorBgwfHU0891bFBASB1fk4GgHTkqMsWaAHypAzPJAKAilQGTd67d29UVVW12l5dXR2NjY0ZTAQA3aAMmgwAuZGjLlugBciTwsGsJ4gvfOEL8dOf/jS+8pWvHHGfZ599Nm644Yb48Y9/HL169Yp3vvOdcfXVV0f//v17cFIA6EYlNvlIV8h2pZqammhqamq1vbGxMV71qld1++cDQI9I4OdkAOC/5ajLnkELQI+544474qabbmp3v/r6+tiyZUt86Utfiptvvjl+9KMfxbx587p9PgDgf9TW1kZDQ0OLbU1NTfHcc8/F0KFDM5oKAAAAyt8ruoL2wIEDsXbt2tiwYUM89dRT0dTUFP37949hw4ZFXV1dnHfeedGnT5/umhWAUmV0i4gdO3bEJz/5yVi/fn2cfPLJbe77i1/8Ih588MG47777YtSoURERMX/+/JgxY0ZceeWVMWzYsB6YOH2aDFDmyuC2TXV1dbF48eLYsmVLnHTSSRER8eCDD0ZExJve9KYsR0uKJgOUuTJoMh2jyQAVIEdd7vAVtNu2bYt3vvOd8fGPfzw2b94cNTU18epXvzr69esXmzZtijlz5sS73vWuePLJJ7tzXgBKUSh0/lWCRx99NPr16xf33HNPvPGNb2xz34ceeihe/epXNy/ORkSceeaZ0atXr3j44YdLmqNSaDJABSilySV2+UgOHjwYTz/9dOzbty8iIt74xjfG6aefHldccUX86le/igceeCCuvfbamDp1qhOm/psmA1SABJvMK6fJABUiR03u8BW08+fPjxEjRsTXvva1GDhwYKv3d+3aFVdccUXMnz8/VqxY0aVDAtBFSjgDafLkyW2+39az8M4999w499xzO/Q5O3bsiOHDh7fYVlVVFYMHD46nnnqqQ79HpdNkgAqQ4FnBTz31VEyePDkWLFgQ06ZNi169esXnP//5uO666+KCCy6I6urqeMc73hFz5szJetRkaDJABUiwybxymgxQIXLU5Q4v0G7YsCHuvPPOwwYuImLQoEExe/bseN/73tdlwwHQxcrgTKK9e/dGVVVVq+3V1dXR2NiYwUTp0WSACpBAkxcuXNji1yNGjIjNmze32HbsscfGsmXLenKssqLJABUggSZTOk0GqBA56nKHF2gHDhwYO3bsiNGjRx9xnyeffDJqamq6ZDAAul6xeLDTX9vWFbJdqaamJpqamlptb2xsjFe96lU9MkPqNBmg/JXSZNKhyQDlT5MrgyYDVIY8dbnDz6B9z3veEx/72Mfiq1/9amzZsqX54HlTU1Ns3bo17rrrrvjEJz4R06ZN67ZhAah8tbW10dDQ0GJbU1NTPPfcczF06NCMpkqLJgNAGjQZANKgyQCUmw5fQfuRj3wkevfuHTfeeGPs2bOn1ftHHXVUvO9974uPfvSjXTogAF2oDO7hX1dXF4sXL44tW7bESSedFBERDz74YEREvOlNb8pytGRoMkAFKIMm0z5NBqgAmlwRNBmgQuSoyx1eoO3Vq1dcdtllcfHFF8djjz0WO3bsiL1790ZNTU3U1tbGmDFjDvvMQAASkuA9/A8ePBg7d+6MgQMHRk1NTbzxjW+M008/Pa644oqYN29e7NmzJ6699tqYOnVqDBs2LOtxk6DJABUgwSbzymkyQAXQ5IqgyQAVIkdd7vAC7SH9+vWLN7zhDd0xCwDdLcEzkJ566qmYPHlyLFiwIKZNmxa9evWKz3/+83HdddfFBRdcENXV1fGOd7wj5syZk/WoydFkgDKWYJPpPE0GKGOaXFE0GaDM5ajLr3iBFoAyVsj+IesLFy5s8esRI0bE5s2bW2w79thjY9myZT05FgD0rASaDACEJgNASnLUZQu0AHmSozOQACBpmgwAadBkAEhHjrrcO+sBAAAAAAAAAPLCFbQAeZKjh6wDQNI0GQDSoMkAkI4cddkCLUCe5OgWEQCQNE0GgDRoMgCkI0ddtkALkCc5OgMJAJKmyQCQBk0GgHTkqMsWaAHyJEeBA4CkaTIApEGTASAdOeqyBVqAHCkWD2Y9AgAQmgwAqdBkAEhHnrrcO+sBAAAAAAAAAPLCFbQAeZKjW0QAQNI0GQDSoMkAkI4cddkCLUCeFPMTOABImiYDQBo0GQDSkaMuu8UxQJ4UCp1/AQBdp5Qm6zIAdB1NBoB0ZNTkQqEQy5Yti4kTJ8a4ceNi5syZsXXr1iPu/8wzz8RVV10Vb37zm+Oss86KK664Inbs2PGKPjPJK2jfUnVc1iO06xfxeNYjtKn2qGOyHqFN/7Z3SNYjtOu3sSfrEdr06lcdnfUI7drVmPbfw30HmrIeoefl6AwkusaNT/4o6xHaVdW3X9YjtOn3p4/KeoR2jfz5b7MeoU2vfzr9P7sGVvXPeoQ2XTvkzVmP0Ka5f/xp1iP0PE2mEzY9e+QDBCn4eKQ9XznYtehdWY/QpnE3PJT1CO367o5fZj1CmwrFYtYj8Kc0mU4Y/79OyXqENq0eXJP1CO0a+/ijWY/QpnlP/TDrEdp1dM1RWY/Qps8NqMt6hDYdU/+1rEfgcDLq8vLly2P16tWxcOHCqK2tjUWLFsWMGTPi3nvvjaqqqlb7X3755XHgwIG47bbbolgsxnXXXRcf/vCH42tf6/g/V66gBQAAAADoYVlcrQMAtNTU1BSrVq2K+vr6mDRpUowZMyaWLl0a27dvj7Vr17baf9euXfHggw/GzJkz47WvfW2MHTs2Lrroovj1r38dzz33XIc/1wItQJ64bRMApMHtFAEgDRk2+dDVOtdff33ceeedUSgUYsaMGdHUdPg7fl1++eXx5JNPxm233Ra33XZbPPnkk/HhD3+4pBkAICkZNHnTpk3x4osvxoQJE5q3DRo0KMaOHRsbNmxotX9NTU0cddRRsWbNmti9e3fs3r07vvGNb8TIkSNj0KBBHf7cJG9xDEA3cesmAEiDJgNAGjJq8qGrdWbNmhWTJk2KiIilS5fGxIkTY+3atTFlypQW+x+6WueWW26J1772tRERcdFFF8Wll14azz33XAwePLiHvwMA6AYldHny5Mltvr9u3brDbt++fXtERAwfPrzF9qFDhza/93JVVVWxcOHCuPbaa+OMM86IXr16xdChQ+P222+P3r07fl2sK2gB8sSVOgCQBlfQAkAaMmpyVlfrAEDSMmjy3r17IyJaPWu2uro6GhsbW+1fLBbjsccei/Hjx8cdd9wR//Iv/xLHHXdcXHrppbF79+4Of64raAHyxAFdAEiDJgNAGkpscrldrQMASSuhy0dqbntqamoi4qW7Wxz664iIxsbG6N+/f6v9v/3tb8ftt98eP/jBD2LAgAEREbFixYp429veFl/72tfiwgsv7NDnqjdAnhQLnX8BAF2nlCbrMgB0nYyanNXVOgCQtAyafOhkqYaGhhbbGxoaYtiwYa32f+ihh2LkyJHNi7MREUcffXSMHDkytmzZ0uHPdQUtAAAAAEAnlNvVOgBAS2PGjIkBAwbE+vXr48QTT4yIl579vnHjxpg+fXqr/Wtra+Nb3/pWNDY2RnV1dURE7NmzJ7Zt2xZ//dd/3eHPdQUtQJ541h0ApMEzaAEgDRk1OaurdQAgaRk0uaqqKqZPnx6LFy+OdevWxaZNm+KKK66I2traOO+88+LgwYPx9NNPx759+yIiYurUqRERcfnll8emTZti06ZNceWVV0Z1dXVMmzatw59rgRYgT9xKEQDS4BbHAJCGjJr88qt1Djl0tU5dXV2r/Wtra2PLli0tbn986Gqdk08+udNzAEBSMvo5ub6+Pt7znvfE3Llz4/zzz48+ffrEypUro1+/fvHUU0/F2WefHffdd19EvPS8+NWrV0exWIwLLrggPvCBD0S/fv1i9erVMXDgwA5/plscA+SJK24AIA2aDABpyKjJL79aZ8iQIXH88cfHokWLWlyts3Pnzhg4cGDU1NTE1KlTY+XKlXH55ZfHRz/60YiIuOmmm17x1ToAkLSMutynT5+YPXt2zJ49u9V7I0aMiM2bN7fYNmrUqFixYkVJn+kKWoA8caUOAKTBFbQAkIYMm5zF1ToAkLQc/ZzsClqAPHG1DgCkQZMBIA0ZNjmLq3UAIGk5+lnZFbQAAAAAAAAAPcQVtAB5kqMzkAAgaZoMAGnQZABIR466bIEWIE+KxawnAAAiNBkAUqHJAJCOHHXZAi1AnuToDCQASJomA0AaNBkA0pGjLlugBciTHAUOAJKmyQCQBk0GgHTkqMsWaAHypJifwAFA0jQZANKgyQCQjhx1uXfWAwAAAAAAAADkxSu6gvb9739/9OrVq0P7fvnLX+7UQAB0oxzdIqLSaTJAmdPkiqHJAGVOkyuGJgNUgBx1+RUt0J599tlx8803x8iRI+MNb3hDd80EQHcpFrOegC6iyQBlTpMrhiYDlDlNrhiaDFABctTlV7RAe/HFF8eAAQNiyZIl8YUvfCFGjBjRXXMB0B1ydAZSpdNkgDKnyRVDkwHKnCZXDE0GqAA56vIrfgbt+973vjjzzDPjxhtv7I55AOhOhULnXyRHkwHKWClN1uXkaDJAGdPkiqLJAGUuR01+RVfQHjJ//vx49NFHu3oWALpbsfxCRds0GaBMaXLF0WSAMqXJFUeTAcpYjrrcqQXaoUOHxtChQ7t6FgDgFdJkAEiDJgNAGjQZgHLQqQVaAMpTsZCfh6wDQMo0GQDSoMkAkI48ddkCLUCelOG9+AGgImkyAKRBkwEgHTnqsgVagDzJ0T38ASBpmgwAadBkAEhHjrpsgRYgT3J0iwgASJomA0AaNBkA0pGjLvfOegAAelCh0PkXANB1SmlyCV0uFAqxbNmymDhxYowbNy5mzpwZW7duPeL+zzzzTFx11VXx5je/Oc4666y44oorYseOHZ3+fABITkZNBgAOI0dNtkALAACQE8uXL4/Vq1fH9ddfH3feeWcUCoWYMWNGNDU1HXb/yy+/PJ588sm47bbb4rbbbosnn3wyPvzhD/fw1AAAAFBZLNAC5EmOzkACgKRlcLVOU1NTrFq1Kurr62PSpEkxZsyYWLp0aWzfvj3Wrl3bav9du3bFgw8+GDNnzozXvva1MXbs2Ljooovi17/+dTz33HMl/g0AgES4ghYA0pGjJlugBciTYrHzLwCg65TS5E52edOmTfHiiy/GhAkTmrcNGjQoxo4dGxs2bGi1f01NTRx11FGxZs2a2L17d+zevTu+8Y1vxMiRI2PQoEGd/tYBICkZNBkAOIIcNblv1gMA0IMyOpOoUCjE5z//+fh//+//xQsvvBB1dXVx7bXXxgknnHDY/Z955pn49Kc/Hffff38Ui8V4y1veEh/72Mdi2LBhPTw5AHSTEps8efLkNt9ft25dq23bt2+PiIjhw4e32D506NDm916uqqoqFi5cGNdee22cccYZ0atXrxg6dGjcfvvt0bu3c30BqBBleMUNAFSsHHXZT9UAeVIodv5VAs+7A4A/UUqTO9nlvXv3RsRLC68vV11dHY2Nja32LxaL8dhjj8X48ePjjjvuiH/5l3+J4447Li699NLYvXt3p2YAgORk0GQA4Ahy1GRX0ALkSbHnz0A69Ly7WbNmxaRJkyIiYunSpTFx4sRYu3ZtTJkypcX+h553d8stt8RrX/vaiIi46KKL4tJLL43nnnsuBg8e3MPfAQB0gxKbfLgrZNtTU1MTES+1+dBfR0Q0NjZG//79W+3/7W9/O26//fb4wQ9+EAMGDIiIiBUrVsTb3va2+NrXvhYXXnhh54YHgJRk8HMyAHAEOeqyK2gB6FaedwcAaTh0a+OGhoYW2xsaGg77GIGHHnooRo4c2bw4GxFx9NFHx8iRI2PLli3dOywAAABUMFfQAuRJCbd66Myz7iI87w4ADiuD2y+NGTMmBgwYEOvXr48TTzwxIl66c8XGjRtj+vTprfavra2Nb33rW9HY2BjV1dUREbFnz57Ytm1b/PVf/3WPzg4A3aYMb4kIABUrR11OcoF2+ZM/yXqEdj397tdkPUKbXv2N32U9Qpuu2/3DrEdoV9/efbIeoU1HV78q6xHadf6r35T1CG36f8/8MusRelwxg4est/W8u+eff77V/i9/3t2MGTPi4MGDsXTp0rj00kvjX//1X1tcxUP3mzd8UtYjtGveUz/MeoQ2nfjQb7MeoV3nDD0t6xHaVNUr7SZHRPzwjxuzHqFNs7f/IOsR2vTJMvizpqtl0eSqqqqYPn16LF68OIYMGRLHH398LFq0KGpra+O8886LgwcPxs6dO2PgwIFRU1MTU6dOjZUrV8bll18eH/3oRyMi4qabborq6uqYNm1aj89P+j+jFBK/HVnvXumf7Ddo9r1Zj9Cmd9SOy3qEdv3XCw3t75Sh2cPPznqEdt345I+yHqFHZdFkyt8jO3+f9Qhtev2z6Tfv08MmZT1Cmzb02p31CO16dF/rCx9ScuHOH2c9Qpv+Yugbsh6hXcf3Tn8NoKvlqctJLtAC0E1KOAOpM8+6i/C8OwA4rIzOCq6vr48DBw7E3LlzY9++fVFXVxcrV66Mfv36xbZt22Ly5MmxYMGCmDZtWgwdOjRWr14dixYtigsuuCB69+4dZ5xxRqxevToGDhyYyfwA0OVydKUOACQvR122QAuQJxlc1fDy590dup3ioV+PHj261f6edwdALmR0pWGfPn1i9uzZMXv27FbvjRgxIjZv3txi26hRo2LFihU9NR4A9LzEr/4HgFzJUZfTv9cBAF2nUOz8q5Ne/ry7Qw49766urq7V/rW1tbFly5ZobGxs3nboeXcnn3xyp+cAgKSU0uQcnVEMAN1OkwEgHTlqsgVaALrVy593t27duti0aVNcccUVLZ539/TTT8e+ffsiImLq1KkREXH55ZfHpk2bYtOmTXHllVd63h0AAAAAABXBAi1AnhQKnX+VoL6+Pt7znvfE3Llz4/zzz48+ffo0P+/uqaeeirPPPjvuu+++iIjm590Vi8W44IIL4gMf+ED069fP8+4AqCylNLnELgMAL6PJAJCOHDXZM2gB8iSjWz143h0A/IkyvP0SAFQkTQaAdOSoyxZoAfIkRw9ZB4CkaTIApEGTASAdOeqyBVqAPMnRGUgAkDRNBoA0aDIApCNHXbZAC5AjxTK8Fz8AVCJNBoA0aDIApCNPXe6d9QAAAAAAAAAAeeEKWoA8ydEtIgAgaZoMAGnQZABIR466bIEWIE9yFDgASJomA0AaNBkA0pGjLlugBciTYn7u4Q8ASdNkAEiDJgNAOnLUZQu0AHmSozOQACBpmgwAadBkAEhHjrpsgRYgR4o5ChwApEyTASANmgwA6chTl3t3dMf//M//jM997nNxww03xI9//ONW7+/evTvmzJnTpcMBAIenywCQBk0GgDRoMgDlpEMLtA8//HBMnTo17r333vjJT34SF198cXz0ox+Npqam5n327dsXa9as6a45AegKhWLnXyRDlwEqQClN1uVkaDJABdDkiqDJABUiR03u0ALtkiVL4m//9m9j7dq18d3vfjduuumm+OlPfxqXXnppHDhwoLtnBKCrFAqdf5EMXQaoAKU0WZeTockAFUCTK4ImA1SIHDW5Qwu0mzdvjn/8x39s/vVf/uVfxhe/+MV4+OGH45prrum24QDoYjk6A6mS6TJABXC1TkXQZIAKoMkVQZMBKkSOmtyhBdoBAwbEM88802Lb6aefHosWLYpvf/vbsWDBgm4ZDoAulqPAVTJdBqgADgZXBE0GqACaXBE0GaBC5KjJHVqgPeecc+K6666LX/7yl7F///7m7W9/+9vj4x//ePzLv/xLzJ8/v9uGBKBrFIvFTr9Ihy4DlL9SmqzL6dBkgPKnyZVBkwEqQ56a3KEF2quuuiqOPfbYOP/88+NnP/tZi/emT58e1157bXz/+9/vlgEBgJZ0GQDSoMkAkAZNBqDc9O3ITkcffXSsWrUq/uu//iuOOeaYVu+/973vjQkTJsTatWu7fEAAulAZ3uqB1nQZoAJockXQZIAKoMkVQZMBKkSOutyhBdpDTjzxxCO+N3LkyLj44otLHgiAbpSjwOWBLgOUMU2uKJoMUMY0uaJoMkCZy1GXX9ECLQDlrZijwAFAyjQZANKgyQCQjjx1uUPPoAWgQhSKnX8BAF2nlCbrMgB0HU0GgHRk1ORCoRDLli2LiRMnxrhx42LmzJmxdevWI+6/f//+WLJkSfP+06dPj8cee+wVfaYFWoA8KZTwAgC6TilN1mUA6DqaDADpyKjJy5cvj9WrV8f1118fd955ZxQKhZgxY0Y0NTUddv958+bF3XffHZ/+9KfjrrvuiiFDhsTMmTPjhRde6PBnWqAFAAAAAOhhWVytAwC01NTUFKtWrYr6+vqYNGlSjBkzJpYuXRrbt2+PtWvXttp/69atcdddd8WnPvWpmDhxYowaNSpuuOGGqKqqit/85jcd/lwLtAA5UiwUO/0CALpOKU3WZQDoOlk2OYurdQAgZVk0edOmTfHiiy/GhAkTmrcNGjQoxo4dGxs2bGi1//333x8DBw6Mt771rS32//73v9/i92iPBVqAPPFcHQBIg+fdAUAaMmpyVlfrAEDSMmjy9u3bIyJi+PDhLbYPHTq0+b2Xe+KJJ+KEE06ItWvXxrRp0+LP//zPY+bMmfH444+/os/t2+mJASg/no8DAGnQZABIQ0ZNbu9qnSlTprTYv62rdQCgYpTQ5cmTJ7f5/rp16w67fe/evRERUVVV1WJ7dXV1PP/886323717d2zZsiWWL18eV199dQwaNChuueWWeO973xv33XdfHHvssR2a1wItQI64JSIApEGTASANpTa5sweDS7la59Zbb40dO3bE2LFj42Mf+1iMGjWqk9MDQFqy+Fm5pqYmIl66u8Whv46IaGxsjP79+7fav2/fvrF79+5YunRpc4OXLl0a55xzTnz961+PGTNmdOhzLdAC5ImrdQAgDZoMAGnIqMlZXa0DAEkroctHOimqPYdOlmpoaIgTTzyxeXtDQ0OMHj261f61tbXRt2/fFidI1dTUxAknnBDbtm3r8OdaoAUAAAAA6ITOHgzO6modAKClMWPGxIABA2L9+vXNC7S7du2KjRs3xvTp01vtX1dXFwcOHIhf//rX8frXvz4iIvbt2xdbt26Nd77znR3+XAu0ADnidooAkAZNBoA0ZNXkrK7WAYCUZdHlqqqqmD59eixevDiGDBkSxx9/fCxatChqa2vjvPPOi4MHD8bOnTtj4MCBUVNTE2eccUa85S1viWuuuSbmz58fgwcPjmXLlkWfPn3i3e9+d4c/t3c3fk8ApKZQwgsA6DqlNFmXAaDrZNTkl1+tc8ihq3Xq6upa7f/yq3UOOXS1zkknndT5QQAgJRn9nFxfXx/vec97Yu7cuXH++edHnz59YuXKldGvX7946qmn4uyzz4777ruvef/Pfe5zceaZZ8Zll10W73nPe2L37t3x5S9/OYYMGdLhz3QFLUCOFB3QBYAkaDIApCGrJmd1tQ4ApCyrLvfp0ydmz54ds2fPbvXeiBEjYvPmzS22DRgwIObNmxfz5s3r9GcmuUD7f4ZPynqEdr36Gz/MeoQ2nTDwf2U9Qptq+lRnPUK7jkp8xl0H9mY9Qrv2R9q37ms6eCDrEXqeg8G8Qj8u7Mx6hHYNrH5V1iO06ZSBx2U9Qrv2Fw9mPUKbXig0Zj1Cu049+visR2jT8VWDsx6hTcuf/0XWI7Tr/3T1b6jJdMLYY05sf6cM/fb5P2Q9Qpuq+iR5CKSFK4acmfUIbUr7J7yXfKeQ9n/X/KawK+sR2nXCoKFZj9CzMmxyfX19HDhwIObOnRv79u2Lurq65qt1tm3bFpMnT44FCxbEtGnTIuKlq3UWL14cl112Wezbty9OP/30V3y1Dl1jUOI/hx4spP8fmx/b/oOsR2jTNcedk/UI7Xow8ePDqf9z+J3tv8x6hHZNHDo26xF6Xtr/2HSp9H86AQAAAACoMFlcrQMApMECLUCOuJ0iAKRBkwEgDZoMAOnIU5ct0ALkSY4CBwBJ02QASIMmA0A6ctRlC7QAOZKnM5AAIGWaDABp0GQASEeeumyBFiBH8hQ4AEiZJgNAGjQZANKRpy5boAXIkTwFDgBSpskAkAZNBoB05KnLvbMeAAAAAAAAACAvXEELkCfFXllPAABEaDIApEKTASAdOeqyBVqAHMnTLSIAIGWaDABp0GQASEeeumyBFiBHioX8nIEEACnTZABIgyYDQDry1GXPoAXIkWKh8y8AoOuU0uRSulwoFGLZsmUxceLEGDduXMycOTO2bt16xP33798fS5Ysad5/+vTp8dhjj3V+AABITFZNBgBay1OTLdAC5Eix2KvTLwCg65TS5FK6vHz58li9enVcf/31ceedd0ahUIgZM2ZEU1PTYfefN29e3H333fHpT3867rrrrhgyZEjMnDkzXnjhhU7PAAApyarJAEBreWqyBVoAAIAcaGpqilWrVkV9fX1MmjQpxowZE0uXLo3t27fH2rVrW+2/devWuOuuu+JTn/pUTJw4MUaNGhU33HBDVFVVxW9+85sMvgMAAACoDJ5BC5Aj5XirBwCoRFk0edOmTfHiiy/GhAkTmrcNGjQoxo4dGxs2bIgpU6a02P/++++PgQMHxlvf+tYW+3//+9/vsZkBoLv5ORkA0pGnLruCFiBHioVenX6VwvPuAKClUprc2S5v3749IiKGDx/eYvvQoUOb33u5J554Ik444YRYu3ZtTJs2Lf78z/88Zs6cGY8//ninPh8AUpRFkwGAw8tTk11BC5AjxWI2n3voeXcLFy6M2traWLRoUcyYMSPuvffeqKqqarX/vHnz4oc//GEsXLgwjjvuuLj55ptj5syZ8e1vfzsGDhyYwXcAAF2r1CZPnjy5zffXrVvXatvevXsjIlq1t7q6Op5//vlW++/evTu2bNkSy5cvj6uvvjoGDRoUt9xyS7z3ve+N++67L4499tgSvgMASENWPycDAK3lqcuvaIG2sbExfve738Upp5wSNTU18dhjj8Xtt98eO3bsiNe85jVxwQUXRG1tbXfNCkCJsjiT6NDz7mbNmhWTJk2KiIilS5fGxIkTY+3ata1up3joeXcrVqyIiRMnRkTEDTfcEFOnTo3f/OY3LW7LmGeaDFDeSm/yK/+ptaamJiJeavOhv454qSn9+/dvtX/fvn1j9+7dsXTp0hg1alREvNTwc845J77+9a/HjBkzOjl7ZdFkgPJWjlfccHiaDFD+8tTlDi/Q/v73v48LL7wwGhoa4rjjjosbbrghLr300jj++OPjlFNOie9973tx9913x+rVq5t/eAcgLVkEzvPuup4mA5S/Upu8bt33XvHXHLq1cUNDQ5x44onN2xsaGmL06NGt9q+trY2+ffu2aElNTU2ccMIJsW3btk5MXXk0GaD85elAcCXTZIDKkKcud/gZtJ/5zGdi3LhxsWbNmjjzzDPjQx/6UPzVX/1VfPOb34ybb745vv3tb8fZZ58dCxYs6M55AcjI5MmT23wdiefddT1NBqAzxowZEwMGDIj169c3b9u1a1ds3Lgx6urqWu1fV1cXBw4ciF//+tfN2/bt2xdbt26Nk046qUdmTp0mA0AaNBmActPhBdoHH3wwLr/88hgzZkxcffXV0djYGNOnT49evV5aze7bt29cfPHF8fDDD3fbsACUpljs/Kuz2nreXWNjY6v9X/68uyuvvDJuueWW6Nu3b7z3ve+NZ555pvODVBBNBih/pTS5s12uqqqK6dOnx+LFi2PdunWxadOmuOKKK6K2tjbOO++8OHjwYDz99NOxb9++iIg444wz4i1veUtcc8018dBDD8V//Md/xNVXXx19+vSJd7/73V34d6N8aTJA+cuiyXQ9TQaoDHlqcodvcVxTU9N8kH3IkCHxd3/3d1FdXd1in127dsXAgQO7dkIAukwpt4hYt25dp77O8+66niYDlL+sbttUX18fBw4ciLlz58a+ffuirq4uVq5cGf369Ytt27bF5MmTY8GCBTFt2rSIiPjc5z4Xixcvjssuuyz27dsXp59+enz5y1+OIUOGZDJ/ajQZoPzl6VaKlUyTASpDnrrc4Stozz777Lj++uvjP/7jPyIiYv78+c0HzguFQtx///0xd+7cePvb3949kwJQsmKxV6dfnfXy5929XENDQwwbNqzV/p531z5NBih/pTS5lC736dMnZs+eHT/72c/iF7/4Rdx6660xYsSIiIgYMWJEbN68uXlxNiJiwIABMW/evHjggQfil7/8ZaxatSpOOeWUkr//SqHJAOUvqybTtTQZoDLkqckdXqCdM2dORESsWLGi1Xvf+c534oMf/GCcdNJJceWVV3bddAB0qWKh86/O8ry7rqfJAOWvlCaX0mW6liYDlD9NrgyaDFAZ8tTkDt/ieMiQIXHnnXfGrl27Wr03YcKEuPfee+M1r3lNlw4HQNcqZHAm0cufdzdkyJA4/vjjY9GiRS2ed7dz584YOHBg1NTUtHje3fz582Pw4MGxbNkyz7t7GU0GKH9ZNJmup8kA5U+TK4MmA1SGPHW5wwu0hwwaNKjVtmOOOSaOOeaYLhkIgMrjeXfdQ5MBIA2aDABp0GQAysUrXqAFoHxldS/+Q8+7mz17dqv3Dj3v7uUOPe9u3rx5PTQhAPSscnw+DgBUIk0GgHTkqcsWaAFypFjIT+AAIGWaDABp0GQASEeeumyBFiBHisWsJwAAIjQZAFKhyQCQjjx12QItQI7k6QwkAEiZJgNAGjQZANKRpy5boAXIkUKO7uEPACnTZABIgyYDQDry1OXeWQ8AAAAAAAAAkBeuoAXIkWKOzkACgJRpMgCkQZMBIB156rIFWoAcydND1gEgZZoMAGnQZABIR566bIEWIEfydA9/AEiZJgNAGjQZANKRpy5boAXIkTzdIgIAUqbJAJAGTQaAdOSpyxZoAXIkT7eIAICUaTIApEGTASAdeepy76wHAAAAAAAAAMgLV9AC5Eie7uEPACnTZABIgyYDQDry1OUkF2g/tePHWY9Q9ra+8MesRyh7e5/8SdYjtGngiElZj9Curzz/VNYjtClHd0tolqd7+NM11u34VdYjlL37n7gt6xHa1f+4iVmPQDf7j75PZj1Cm3pF/vqkyXTGqKpjsx6hTY0D92c9Qpve3n9k1iO0a2HD/VmP0KZy+PP6pEHDsh6hTd/e8cusR2jXHUPOyXqEHqXJdEbTwQNZj9CmPU37sh6hXS/cfnHWI7TplIu/mvUI7dqzvzHrEdo0uOaorEdo03P7Xsx6hHbtK6b9Z013yFOXk1ygBaB75OkMJABImSYDQBo0GQDSkacuW6AFyJE8XjUMACnSZABIgyYDQDry1GULtAA5kqczkAAgZZoMAGnQZABIR5663DvrAQAAAAAAAADywhW0ADmSp4esA0DKNBkA0qDJAJCOPHXZAi1AjhSyHgAAiAhNBoBUaDIApCNPXbZAC5AjxcjPGUgAkDJNBoA0aDIApCNPXbZAC5AjhWLWEwAAEZoMAKnQZABIR566bIEWIEcKOToDCQBSpskAkAZNBoB05KnLvbMeAAAAAAAAACAvXEELkCN5uoc/AKRMkwEgDZoMAOnIU5ct0ALkSCHrAQCAiNBkAEiFJgNAOvLUZQu0ADmSpzOQACBlmgwAadBkAEhHnrpsgRYgR/J0BhIApEyTASANmgwA6chTly3QAuRIngIHACnTZABIgyYDQDry1OXeWQ8AAAAAAAAAkIVCoRDLli2LiRMnxrhx42LmzJmxdevWDn3tPffcE6NHj45t27a9os8seYH2oosuioaGhlJ/GwB6QDF6dfpF+jQZoHyU0mRdLg+6DFAesmxyFgeD80iTAcpHVk1evnx5rF69Oq6//vq48847o1AoxIwZM6KpqanNr/vDH/4Q8+fP79RndugWx2vWrDnie+vXr49vfvObMWTIkIiImDp1aqcGAaD7FRzPLXuaDFAZNLky6DJA+cuyyYcOBi9cuDBqa2tj0aJFMWPGjLj33nujqqrqiF9XysHgSqXJAJUhiy43NTXFqlWrYtasWTFp0qSIiFi6dGlMnDgx1q5dG1OmTDns1xUKhZg9e3acdtpp8cADD7ziz+3QAu11110X+/bti4iIYrHY6v0bb7wxIiJ69eolcAAJK7jipuxpMkBl0OTKoMsA5S+rJmd1MLhSaTJAZciiy5s2bYoXX3wxJkyY0Lxt0KBBMXbs2NiwYcMRm7xixYrYv39/XHbZZd23QHv33XfHrFmzYtCgQbFw4cIYNmxY83vjx4+Pe+65J0444YRX/OEA9KzWP6JQbjQZoDJocmXQZYDyl1WTszoYXKk0GaAylNLlyZMnt/n+unXrDrt9+/btERExfPjwFtuHDh3a/N6f+tWvfhWrVq2Kr33ta7Fjx45OTNvBZ9COHDkyvvrVr8brX//6ePe73x333Xdfpz4MgGwVSniRBk0GqAylNFmX06HLAOWv1CZPnjy5zdeRlHIweNGiRdGnT5/Of9MVSJMBKkMWPyfv3bs3IqLV4wWqq6ujsbGx1f579uyJWbNmxaxZs+Lkk0/u9Od26AraiIi+ffvGlVdeGRMnToxrrrkmvv/978cnP/nJTn8wANA5mgwA6dBlADqjrYPBzz//fKv9//RgcGev1qlkmgyQb0e6QrY9NTU1EfHS4wcO/XVERGNjY/Tv37/V/jfccEOMHDky/uEf/qFzg/63Di/QHlJXVxdr1qyJ6667LqZMmRL79+8vaQAAek6hl+fdVRJNBihfmlx5dBmgPJXa5HI7GJwHmgxQvrL4WfnQ3SwaGhrixBNPbN7e0NAQo0ePbrX/XXfdFVVVVTF+/PiIiDh48GBEREyZMiUuueSSuOSSSzr0ua94gTbipechLFmyJNasWRN33313VFdXd+a3AaCHed5d5dFkgPKkyZVJlwHKT1ZNzupgcF5oMkB5yqLLY8aMiQEDBsT69eubm7xr167YuHFjTJ8+vdX+a9eubfHrRx55JGbPnh233nprnHrqqR3+3E4t0B4yderUmDp1aim/BQA9yDPrKpcmA5QXTa5sugxQPrJqclYHg/NGkwHKSxZdrqqqiunTp8fixYtjyJAhcfzxx8eiRYuitrY2zjvvvDh48GDs3LkzBg4cGDU1NXHSSSe1+PpDz44/7rjjYvDgwR3+3JIWaAEoLwV3UwSAJGgyAKQhqyZndTAYAFKWVZfr6+vjwIEDMXfu3Ni3b1/U1dXFypUro1+/frFt27aYPHlyLFiwIKZNm9Zln2mBFiBHCuFoMACkQJMBIA1ZNjmLg8EAkLKsutynT5+YPXt2zJ49u9V7I0aMiM2bNx/xa88666w23z8SC7QAAAAAAD0si4PBAEAaLNAC5EgWD1kHAFrTZABIgyYDQDry1GULtAA54nl3AJAGTQaANGgyAKQjT13unfUAAPScQgkvAKDrlNLkUrpcKBRi2bJlMXHixBg3blzMnDkztm7d2qGvveeee2L06NGxbdu2EiYAgLRk1WQAoLU8NdkCLUCOFEt4AQBdp5Qml9Ll5cuXx+rVq+P666+PO++8MwqFQsyYMSOampra/Lo//OEPMX/+/BI+GQDSlFWTAYDW8tRkC7QAOVLo1flXSZ/rah0AaKGUJne2y01NTbFq1aqor6+PSZMmxZgxY2Lp0qWxffv2WLt27ZFnLRRi9uzZcdppp3XyuwWAdGXRZADg8PLUZAu0AHQ7V+sAQPY2bdoUL774YkyYMKF526BBg2Ls2LGxYcOGI37dihUrYv/+/XHxxRf3xJgAAABQ8fpmPQAAPSeLe/Efulpn1qxZMWnSpIiIWLp0aUycODHWrl0bU6ZMOezXvfxqnQceeKAHJwaA7ldqkydPntzm++vWrWu1bfv27RERMXz48Bbbhw4d2vzen/rVr34Vq1atiq997WuxY8eOTk4LAOkqx2fWAUClylOXXUELkCNZPGTd1ToA0FopTe5sl/fu3RsREVVVVS22V1dXR2NjY6v99+zZE7NmzYpZs2bFySef3MlPBYC0ZdFkAODw8tRkV9AC5Egxg3vxu1oHAFortcmHu0K2PTU1NRHx0t0tDv11RERjY2P079+/1f433HBDjBw5Mv7hH/6h84MCQOKy+DkZADi8PHU5yQXagVWtDw6k5l/7j896hDbVF5/IeoQ2PbnnmaxHaNdxo/531iO0adTRw9vfKWMHi2mft1LVO8k/ArtVKf+PdOZWihFtX63z/PPPt9r/T6/WsUCbrdcPOTnrEdq1r7A/6xHadNpr/y7rEdpV07eq/Z0y9Lb/dVrWI7SrGMWsR2jTA8/9LusR2vSuIa/PeoQel8V/JR06WaqhoSFOPPHE5u0NDQ0xevToVvvfddddUVVVFePHv/Szz8GDByMiYsqUKXHJJZfEJZdc0gNT83L37vhF1iO06a+Gjct6hDatbFif9QjtGn30iKxHaNOjz27JeoR23Vl9UtYjtOl9vdK/md35z/ww6xHa9P918e+X9pELUnWwkPY/ORuGvynrEdo18qJ/zXqENj3fuCfrEdr18aFnZz1Cm77R9F9Zj9Cm3x78Q9YjtOsPe/+Y9Qg9Lu0/XbtW/lYnAOhRrtYBgDSMGTMmBgwYEOvXr29eoN21a1ds3Lgxpk+f3mr/tWvXtvj1I488ErNnz45bb701Tj311B6ZGQAAACqRBVqAHCnlDKTO3EoxwtU6AHA4WZwVXFVVFdOnT4/FixfHkCFD4vjjj49FixZFbW1tnHfeeXHw4MHYuXNnDBw4MGpqauKkk1pehXbo0QTHHXdcDB48OIPvAAC6Xp6u1AGA1OWpyxZoAXIkixtwuloHAFrL6qbY9fX1ceDAgZg7d27s27cv6urqYuXKldGvX7/Ytm1bTJ48ORYsWBDTpk3LaEIA6FlpP6gCAPIlT122QAuQI4UMHrLuah0AaC2LJkdE9OnTJ2bPnh2zZ89u9d6IESNi8+bNR/zas846q833AaAcZdVkAKC1PHXZAi1AjmR1iwhX6wBAS3m6bRMApEyTASAdeeqyBVqAHMkqcK7WAYCW8vRDJwCkTJMBIB156nLvrAcAAAAAAAAAyAtX0ALkSJ4esg4AKdNkAEiDJgNAOvLUZQu0ADmSp4esA0DKNBkA0qDJAJCOPHXZAi1AjuTpHv4AkDJNBoA0aDIApCNPXbZAC5AjebpFBACkTJMBIA2aDADpyFOXLdAC5EghV4kDgHRpMgCkQZMBIB156nLvrAcAAAAAAAAAyAtX0ALkSJ7u4Q8AKdNkAEiDJgNAOvLUZQu0ADmSnxtEAEDaNBkA0qDJAJCOPHXZAi1AjuTpDCQASJkmA0AaNBkA0pGnLnf4GbRr1qyJpqamFtseeOCBuOiii+Kv//qv46qrrorHH3+8ywcEoOsUenX+RTo0GaD8ldJkXU6HJgOUP02uDJoMUBny1OQOL9DOmTMnXnjhheZf/+QnP4kPfOADUSwW4+yzz46GhoaYNm1a/PznP++WQQEoXSGKnX6RDk0GKH+lNFmX06HJAOVPkyuDJgNUhjw1ucO3OC4WW35zt9xyS1x44YVxzTXXNG9bsGBBLF68OFavXt11EwIALWgyAKRBkwEgDZoMQLnp8BW0f2rLli3xrne9q8W2v//7v4+NGzeWPBQA3aNYwot0aTJA+SmlybqcLk0GKD+aXJk0GaA85anJHb6CtlevljdwHjlyZOzevbvFtp07d8bAgQO7ZjIAulyeHrJeyTQZoPxpcmXQZIDyp8mVQZMBKkOeuvyKbnE8efLkOPnkk2PUqFHRt2/fWLhwYdx5551RVVUVGzZsiPnz58db3/rW7pwXgBKU4734aU2TAcqfJlcGTQYof5pcGTQZoDLkqcsdXqD90Y9+FJs3b47f/va3sXnz5nj22Wfj97//fRw8eDAiIi655JIYNWpUXHXVVd02LAClyU/eKpsmA5Q/Ta4MmgxQ/jS5MmgyQGXIU5c7vEA7bNiwGDZsWIuzjA4ePBh9+vSJiIivfvWrMWrUqFa3kwAgHXm6RUQl02SA8qfJlUGTAcqfJlcGTQaoDHnqcocXaA/nUOAiIk455ZSShwEAOkeTASANmgwAadBkAFJW0gItAOUlT/fwB4CUaTIApEGTASAdeeqyBVqAHMlP3gAgbZoMAGnQZABIR566bIEWIEfydA9/AEiZJgNAGjQZANKRpy5boAXIkWKuzkECgHRpMgCkQZMBIB156rIFWoAcydMZSACQMk0GgDRoMgCkI09d7p31AAAAAAAAAAB54QpagBwp5OgWEQCQMk0GgDRoMgCkI09dtkALkCP5yRsApE2TASANmgwA6chTly3QAuRIns5AAoCUaTIApEGTASAdeeqyBVqAHMnTQ9YBIGWaDABp0GQASEeeumyBFiBHijk6AwkAUqbJAJAGTQaAdOSpy72zHgAAAAAAAAAgL1xBC5AjebpFBACkTJMBIA2aDADpyFOXk1ygrelblfUI7fp6/7Qvsx5TGJb1CG06uu+rsh6hXVv2NGQ9Qps2P7st6xHaNbjmqKxHaNOJRw3NeoQel6dbRNA1hvYblPUI7fr5rt9nPUKbntu7O+sR2vXoqNdnPUKbrnox6wnat+7p32Q9Qptq+vTLeoQ23f7kA1mP0K7buvj302Q6o0/vtG+Cdc9TD2c9QptOHJT+f/+v//WXsx6hTQNGnJP1CO16y9MPZj1Cm8rhT/9j+g/IeoQepcl0xtBXDc56hDad9vBNWY/Qrj8eNzHrEcre4mfS/jnqQOFg1iO0qap3kstjLTy5e2fWI/S4PHU5/X8CAegyeToDCQBSpskAkAZNBoB05KnLFmgBcqRQzM8ZSACQMk0GgDRoMgCkI09dtkALkCP5yRsApE2TASANmgwA6chTl9N+gA0AAAAAAABABbFAC5AjhSh2+gUAdJ1SmqzLANB1NBkA0pFVkwuFQixbtiwmTpwY48aNi5kzZ8bWrVuPuP/vfve7uOiii+Kss86KCRMmRH19fTz55JOv6DMt0ALkSLGE/wEAXaeUJusyAHSdLJucxcFgAEhZVk1evnx5rF69Oq6//vq48847o1AoxIwZM6KpqanVvs8++2x84AMfiJqamvjKV74SX/ziF2Pnzp0xY8aMaGxs7PBnWqAFyJFCCS8AoOuU0mRdBoCuk2WTszgYDAApy6LJTU1NsWrVqqivr49JkybFmDFjYunSpbF9+/ZYu3Ztq/2/973vxZ49e+LGG2+MU089NV73utfFokWL4vHHH4+f//znHf5cC7QAOeK2TQCQBrdTBIA0ZNXkrA4GA0DKsmjypk2b4sUXX4wJEyY0bxs0aFCMHTs2NmzY0Gr/CRMmxPLly6OmpqZ5W+/eLy237tq1q8Of27fTEwNQdtwSEQDSoMkAkIasmtzeweApU6a02L+rDgYDQMpK6fLkyZPbfH/dunWH3b59+/aIiBg+fHiL7UOHDm1+7+VGjBgRI0aMaLHt1ltvjZqamqirq+vwvBZoAQAAAAA6odwOBgMALe3duzciIqqqqlpsr66ujueff77dr//KV74St99+e8ydOzeGDBnS4c+1QAuQI55ZBwBp0GQASENWTc7qYDAApKyULh/ppKj2HLo7RVNTU4s7VTQ2Nkb//v2P+HXFYjFuvvnmuOWWW+JDH/pQvP/9739Fn2uBFiBHikW3UwSAFGgyAKSh1CaX28FgAEhZFj8rH7qbRUNDQ5x44onN2xsaGmL06NGH/Zr9+/fHnDlz4pvf/GbMmTMnLrzwwlf8ub07NS0AZSmLh6wDAK2V0uRSulwoFGLZsmUxceLEGDduXMycOTO2bt16xP1/97vfxUUXXRRnnXVWTJgwIerr6+PJJ5/s9OcDQGqyavLLDwa/XENDQwwbNuywX7N///6YPXt2rFixIubMmROXX355pz8fAFKURZPHjBkTAwYMiPXr1zdv27VrV2zcuPGIjxG4+uqr4zvf+U4sWbKkU4uzERZoAXKlUMKrpM91MBgAWiilyaV0efny5bF69eq4/vrr484774xCoRAzZsyIpqamVvs+++yz8YEPfCBqamriK1/5Snzxi1+MnTt3xowZM6KxsbGEKQAgHVk1OauDwQCQsiyaXFVVFdOnT4/FixfHunXrYtOmTXHFFVdEbW1tnHfeeXHw4MF4+umnY9++fRERcffdd8d9990XV1xxRZx55pnx9NNPN78O7dMRFmgBcqRYwv9K4WAwALRUSpM72+WmpqZYtWpV1NfXx6RJk2LMmDGxdOnS2L59e6xdu7bV/t/73vdiz549ceONN8app54ar3vd62LRokXx+OOPx89//vNS/xYAQBKyaHJEdgeDASBlWR2/rq+vj/e85z0xd+7cOP/886NPnz6xcuXK6NevXzz11FNx9tlnx3333RcREd/85jcjIuLGG2+Ms88+u8Xr0D4d4Rm0AHSrQweDZ82aFZMmTYqIiKVLl8bEiRNj7dq1MWXKlBb7v/xg8KHn8CxatCgmTZoUP//5z2PChAk9/S0AQEXYtGlTvPjiiy1aOmjQoBg7dmxs2LChVZMnTJgQy5cvb/FcvN69XzrHd9euXT0zNABUsPr6+jhw4EDMnTs39u3bF3V1dc0Hg7dt2xaTJ0+OBQsWxLRp01ocDL7xxhtb/D6H9gEAOqdPnz4xe/bsmD17dqv3RowYEZs3b27+9apVq7rkM1/RAu0jjzwS69evj4suuigiIh544IH40pe+FNu2bYsTTzwx/umf/inOOOOMLhkMgK6XxbNkHQzuHpoMUN5KbfLkyZPbfH/dunWttm3fvj0i/ueZd4cMHTq0+b2XGzFiRIwYMaLFtltvvTVqamqOeOvFPNJkgPKWxc/Jh2RxMLiSaTJA+cuyyz2tw7c4/s53vhPnn39+PPjggxER8YMf/CA+8IEPRLFYjHPOOSf2798fF1xwQfzgBz/otmEBKE2xWOz0a/LkyW2+jqQzB4Pf/OY3t9jmYHBLmgxQ/kppcrHYuR9Y9+7dGxEv3VLx5aqrqzv0GIGvfOUrcfvtt8esWbNiyJAhnZqh0mgyQPnLosl0PU0GqAx5anKHr6D9/Oc/H/X19XHJJZdERMQtt9wSl1xySXz0ox9t3ueWW26JZcuWxdve9raunxSAkpXysPTOautg8PPPP9/u1x86GDx37lwHg/+bJgOUv1KbfLgrZNtz6O4UTU1NLe5U0djYGP379z/i1xWLxbj55pvjlltuiQ996EPx/ve//5UPXKE0GaD8ZfFzMl1PkwEqQ5663OEF2v/6r/+Kd77znc2/3rZtW/zlX/5li32mTJkSt9xyS9dNB0CXKuVh6Z05EBzhYHB30GSA8ldKkzvr0N0sGhoa4sQTT2ze3tDQEKNHjz7s1+zfvz/mzJkT3/zmN2POnDlx4YUX9sSoZUOTAcpfFk2m62kyQGXIU5c7fIvjE044Ie6///7mX7/2ta+NTZs2tdjnV7/6VQwbNqzrpgOgSxWi2OlXZ738YPDLNTQ0HLEZ+/fvj9mzZ8eKFStizpw5cfnll3f68yuRJgOUv1Ka3NkujxkzJgYMGBDr169v3rZr167YuHHjER8jcPXVV8d3vvOdWLJkicXZw9BkgPKXRZPpepoMUBny1OQOX0E7c+bMmDt3bmzbti2mTJkSl156aXzsYx+LxsbGeM1rXhOPPPJI/PM//3Ncdtll3TkvAGXm5QeDD12tc+hg8PTp0w/7NVdffXX8+7//eyxZsqTFGbC8RJMB6IyqqqqYPn16LF68OIYMGRLHH398LFq0KGpra+O8886LgwcPxs6dO2PgwIFRU1MTd999d9x3331x9dVXx5lnnhlPP/108+91aJ+802QASIMmA1BuOrxAO3Xq1OjVq1csW7Ys/u///b/Rq1evKBaL8clPfjIiIo466qiYMWOGs6oBEpbFw9IdDO56mgxQ/rJockREfX19HDhwIObOnRv79u2Lurq6WLlyZfTr1y+2bdsWkydPjgULFsS0adPim9/8ZkRE3HjjjXHjjTe2+H0O7ZN3mgxQ/rJqMl1LkwEqQ5663OEF2oiId7/73fHud787nnjiiXjiiSdi9+7d0bdv36itrY3TTjstqquru2tOALpAVrd6cDC462kyQHnLqsl9+vSJ2bNnx+zZs1u9N2LEiNi8eXPzr1etWtWTo5UtTQYob+V4S0QOT5MByl+euvyKFmgPGTlyZIwcObKrZwGgm2X1kHUHg7uPJgOUp6yaTPfRZIDypMmVR5MByleeutypBVoAylMhR7eIAICUaTIApEGTASAdeeqyBVqAHMlP3gAgbZoMAGnQZABIR5663DvrAQAAAAAAAADywhW0ADmSp4esA0DKNBkA0qDJAJCOPHXZAi1AjuQpcACQMk0GgDRoMgCkI09dtkALkCPFHD1kHQBSpskAkAZNBoB05KnLFmgBciRPZyABQMo0GQDSoMkAkI48ddkCLUCOFHMUOABImSYDQBo0GQDSkacu9856AAAAAAAAAIC8cAUtQI7k6R7+AJAyTQaANGgyAKQjT122QAuQI3m6hz8ApEyTASANmgwA6chTly3QAuRIns5AAoCUaTIApEGTASAdeepykgu0DS8+l/UI7fri7vuzHqFNz1xwWtYjtOnPvvpk1iO067l9L2Y9Qtnb1bgn6xHadO6QE7Ieocfl6QwkusYPn/5N1iO0q1AoZD1Cm7a++TVZj9CuiY89nfUIbXri+e1Zj9Cufn2S/M/qZq87+qSsR2hT3169sx6hx2kylahX1gO048ndz2Q9QrsaP3NV1iO06f8bVpf1CO36t+0PZj1Cm84ZmvbxmoiIHzc8mvUIPUqT6Yxn972Q9QhtGnfa+VmP0K6RR9dmPUKbyuHn0DccfXLWI7TpF8/+PusR2rTgmDdnPUK7VheeynqEHpenLufvSAgAAAAAAABARtI+1R+ALlXM0RlIAJAyTQaANGgyAKQjT122QAuQI4Uc3cMfAFKmyQCQBk0GgHTkqcsWaAFyJE9nIAFAyjQZANKgyQCQjjx12QItQI7k6QwkAEiZJgNAGjQZANKRpy5boAXIkTydgQQAKdNkAEiDJgNAOvLU5d5ZDwAAAAAAAACQF66gBciRPN0iAgBSpskAkAZNBoB05KnLFmgBciRPt4gAgJRpMgCkQZMBIB156rIFWoAcydMZSACQMk0GgDRoMgCkI09dtkALkCN5OgMJAFKmyQCQBk0GgHTkqcsWaAFypFgsZD0CABCaDACp0GQASEeeutw76wEAAAAAAAAA8sIVtAA5UsjRLSIAIGWaDABp0GQASEeeumyBFiBHijl6yDoApEyTASANmgwA6chTly3QAuRIns5AAoCUaTIApEGTASAdeepyh59B+xd/8RexZs2abhwFgO5WLBY7/SIdmgxQ/kppsi6nQ5MByp8mVwZNBqgMeWpyhxdot27dGh//+MfjE5/4ROzatas7ZwKgmxSKxU6/SIcmA5S/Upqsy+nQZIDyp8mVQZMBKkOemtzhBdqIiGXLlsXPfvaz+N//+3/H7bffHk1NTd01FwDQBk0GgDRoMgCkQZMBKCevaIF2/Pjx8a1vfSv+9m//NhYtWhTnnntufPazn43f/va33TUfAF2oWML/SIsmA5S3Upqsy2nRZIDypsmVQ5MByl+emtz3lX5B//7948orr4wLL7wwVq9eHd/4xjfii1/8Yhx77LExevToGDx4cCxZsqQ7ZgWgROV4L36OTJMBypcmVxZNBihfmlxZNBmgvOWpyx1eoO3Vq1eLXw8ZMiQuu+yyuOyyy2LTpk3x8MMPx8aNG+Ppp5/u8iEB6BqFMjyTiNY0GaD8aXJl0GSA8qfJlUGTASpDnrrc4QXatlatx4wZE2PGjOmSgQDoPnk6A6mSaTJA+dPkyqDJAOVPkyuDJgNUhjx1ucMLtF/+8pfj6KOP7s5ZAOhmhRwFrpJpMkD50+TKoMkA5U+TK4MmA1SGPHW5wwu0Z555ZnfOAQB0kCYDQBo0GQDSoMkAlJsOL9ACUP7ydIsIAEiZJgNAGjQZANKRpy5boAXIkTw9ZB0AUqbJAJAGTQaAdOSpyxZoAXIkT2cgAUDKNBkA0qDJAJCOPHXZAi1AjuTpIesAkDJNBoA0aDIApCNPXbZAC5AjxRzdIgIAUqbJAJAGTQaAdOSpy72zHgAAAAAAAAAgL1xBC5AjebpFBACkTJMBIA2aDADpyFOXLdAC5EieHrIOACnTZABIgyYDQDry1GW3OAbIkWIJ/ytFoVCIZcuWxcSJE2PcuHExc+bM2Lp16xH3f/bZZ+Oqq66Kurq6OPPMM+O6666LvXv3ljQDAKSklCaX0mVNBoCWsmpyhC4DwJ/KU5Mt0ALkSLFY7PSrFMuXL4/Vq1fH9ddfH3feeWcUCoWYMWNGNDU1HXb/+vr62LJlS3zpS1+Km2++OX70ox/FvHnzSpoBAFJSSpNL6bImA0BLWTU5QpcB4E/lqckWaAFyJIvANTU1xapVq6K+vj4mTZoUY8aMiaVLl8b27dtj7dq1rfb/xS9+EQ8++GB85jOfidNOOy0mTJgQ8+fPj2984xuxY8eOUr59AEhGFgeDNRkAWstqgVaXAaC1PDXZAi0A3WrTpk3x4osvxoQJE5q3DRo0KMaOHRsbNmxotf9DDz0Ur371q2PUqFHN284888zo1atXPPzwwz0yMwBUIk0GgHToMgCkIasm9y1tbADKSSk3epg8eXKb769bt+6w27dv3x4REcOHD2+xfejQoc3vvdyOHTta7VtVVRWDBw+Op5566pWMDADJKu3mS53rsiYDQGtZNDlClwHgcPJ0/DrJBdqmxm1Zj0A3++MXs54A8ulA0x86/bXtBe5IDj0cvaqqqsX26urqeP755w+7/5/ue2j/xsbGTs1A5zXu25r1CPSA32U9AORQKU2O6FyXNbn8vbjnP7MegZy7fVHWE7Tv9qwHoOxk0eQIXS53z+7+j6xHAHLgkqwHyECejl8nuUALQHqOdIZRe2pqaiLipXv5H/rriIjGxsbo37//Yfc/3MPXGxsb41WvelWnZgCAStOZLmsyAHQ9PysDQBrKrcmeQQtAtzp0u4eGhoYW2xsaGmLYsGGt9q+trW21b1NTUzz33HMxdOjQ7hsUACqcJgNAOnQZANKQVZMt0ALQrcaMGRMDBgyI9evXN2/btWtXbNy4Merq6lrtX1dXF9u3b48tW7Y0b3vwwQcjIuJNb3pT9w8MABVKkwEgHboMAGnIqslucQxAt6qqqorp06fH4sWLY8iQIXH88cfHokWLora2Ns4777w4ePBg7Ny5MwYOHBg1NTXxxje+MU4//fS44oorYt68ebFnz5649tprY+rUqYc9YwkA6BhNBoB06DIApCGrJvcqFovFbvy+ACAOHjwYn/3sZ+Puu++Offv2RV1dXVx77bUxYsSI2LZtW0yePDkWLFgQ06ZNi4iIZ555Jq677rr4yU9+EtXV1fGOd7wj5syZE9XV1Rl/JwBQ3jQZANKhywCQhiyabIEWAAAAAAAAoId4Bi0AAAAAAABAD7FACwAAAAAAANBDLNACAAAAAAAA9BALtAAAAAAAAAA9xAItAAAAAAAAQA+xQAsAAAAAAADQQyp6gbZQKMSyZcti4sSJMW7cuJg5c2Zs3bo167EO6wtf+EK8//3vz3qMVp577rm49tpr461vfWucfvrpcf7558dDDz2U9VjNnnnmmZg9e3a8+c1vjvHjx8dFF10Ujz/+eNZjHdYTTzwR48ePj7vvvjvrUVrYsWNHjB49utUrpTnXrFkTf/VXfxWvf/3r453vfGd8+9vfznqkZuvXrz/s37/Ro0fH5MmTsx4PklFOTY5Is8ua3HU0uTSpdlmToWM0uXSpNzlCl0ulyaXRZOgYTS6dJnetFJscUR5dTrXJEbqcsr5ZD9Cdli9fHqtXr46FCxdGbW1tLFq0KGbMmBH33ntvVFVVZT1eszvuuCNuuummOOOMM7IepZUrr7wynn766fjsZz8bxx57bHzlK1+JD37wg/H1r389/uzP/izr8eLDH/5wFAqFuPXWW+Ooo46Km2++OS688MJYu3Zt9O/fP+vxmu3fvz9mzZoVe/bsyXqUVjZt2hTV1dXxve99L3r16tW8feDAgRlO9T++8Y1vxCc+8Yn4+Mc/HhMnToxvfetbceWVV0ZtbW2MHz8+6/Fi/Pjx8dOf/rTFtl/+8pfxkY98JC699NKMpoL0lEuTI9LtsiZ3DU0uTcpd1mToGE0uXepNjtDlUmlyaTQZOkaTS6fJXSfVJkek3+WUmxyhy0krVqjGxsbi+PHji3fccUfztueff774hje8oXjvvfdmONn/2L59e/Hiiy8ujhs3rviOd7yjOH369KxHauE///M/i6eeemrxoYceat5WKBSKb3/724s33XRThpO95LnnniteeeWVxc2bNzdve+yxx4qnnnpq8ZFHHslwstaWLFlS/Md//MfiqaeeWrzrrruyHqeFW2+9tfiud70r6zEOq1AoFN/2trcVFy5c2GL7P/3TPxVXrFiR0VRte/HFF4tve9vbih/72MeyHgWSUQ5NLhbT7rImdx1N7rxy67ImQ2uaXLrUm1ws6nJX0OSupcnQmiaXTpO7VqpNLhbT7nK5NblY1OWUVOwtjjdt2hQvvvhiTJgwoXnboEGDYuzYsbFhw4YMJ/sfjz76aPTr1y/uueeeeOMb35j1OK0cc8wxceutt8brX//65m29evWKXr16xa5duzKc7CVHH310LFmyJE499dSIiNi5c2d86Utfitra2jjllFMynu5/bNiwIb761a/GwoULsx7lsDZv3hyjRo3KeozDeuKJJ+IPf/hDvOtd72qxfeXKlXHxxRdnNFXbVqxYEXv37o1rrrkm61EgGeXQ5Ii0u6zJXUOTS1NuXdZkaE2TS5d6kyN0uStoctfSZGhNk0unyV0n5SZHpN3lcmtyhC6npGJvcbx9+/aIiBg+fHiL7UOHDm1+L2vnnntunHvuuVmPcUSDBg2Kc845p8W27373u7Fly5b4+Mc/ntFUh/d//s//iX/7t3+LqqqquOWWW+JVr3pV1iNFRMSuXbvi6quvjrlz57b6ZzEVv/3tb+OYY46J973vffHEE0/ESSedFB/60IfirW99a9ajxRNPPBEREXv27IkPfvCDsXHjxhgxYkR86EMfSvLfnUP/kXXVVVfF4MGDsx4HklEOTY5Iu8uaXDpNLl05dVmT4fA0uXTl1OQIXe4sTe46mgyHp8ml0+SukXqTI9Lucjk1OUKXU1OxV9Du3bs3IqLV/fqrq6ujsbExi5HK3s9//vOYM2dOnHfeeTFp0qSsx2nhggsuiLvuuiumTJkSH/7wh+PRRx/NeqSIiJg3b16MHz++1Rk0qThw4ED8/ve/j+effz4+8pGPxK233hrjxo2Liy66KH72s59lPV7s3r07IiKuueaamDJlSqxatSr+/M//PC699NIk5vtTq1evjoEDB8bf//3fZz0KJEWTu54mv3KaXLpy6rImw+FpctdLuckRutwZmty1NBkOT5O7niZ3TspNjki/y+XU5AhdTk3FXkFbU1MTERFNTU3Nfx0R0djYmNTDt8vF9773vZg1a1acfvrpsXjx4qzHaeXQLSE+9alPxSOPPBK33357LFiwINOZ1qxZEw899FDce++9mc7Rlr59+8b69eujT58+zf+evO51r4vf/e53sXLlyha3WclCv379IiLigx/8YPzN3/xNRES89rWvjY0bN8Ztt92W+Xx/as2aNTF16tQWf+YAmtzVNPmV0+SuUU5d1mQ4PE3uWqk3OUKXO0OTu5Ymw+FpctfS5M5JvckR6Xe5nJococupqdgraA9djt/Q0NBie0NDQwwbNiyLkcrW7bffHh/5yEfibW97W6xYsSKqq6uzHikiXroc/1vf+lYcOHCgeVvv3r3jlFNOafX/exbuuuuueOaZZ2LSpEkxfvz4GD9+fEREfPKTn4wZM2ZkPN3/OOqoo1r9gfya17wmduzYkdFE/+PQv6uHntNwyCmnnBLbtm3LYqQj2rRpU2zdujXZs80gS5rcdTS5czS5a5RLlzUZjkyTu06qTY7Q5a6gyV1Dk+HINLnraHLnlUOTI9Lucrk0OUKXU1SxC7RjxoyJAQMGxPr165u37dq1KzZu3Bh1dXUZTlZeVq9eHddff328733vi89+9rOtbruRpT/+8Y9x5ZVXtrhVwP79+2Pjxo1JPDR88eLFcd9998WaNWuaXxER9fX18alPfSrb4f7b7373uzj99NNb/HsSEfGb3/wmiQfVn3baaXHUUUfFI4880mL7b3/72zjxxBMzmurwHnrooTj22GNjzJgxWY8CydHkrqHJnafJXaNcuqzJcGSa3DVSbnKELpdKk7uOJsORaXLX0OTSpN7kiPS7XC5NjtDlFFXsLY6rqqpi+vTpsXjx4hgyZEgcf/zxsWjRoqitrY3zzjsv6/HKwhNPPBGf/vSn4y/+4i/i4osvjj/+8Y/N79XU1MTAgQMznO6ls1Le+ta3xg033BA33HBDHH300fGFL3whdu3aFRdeeGGms0XEEc92O/bYY5M5E27UqFHxZ3/2ZzF//vy47rrr4phjjol/+7d/i1/+8pdx1113ZT1e1NTUxIwZM+Kf//mfY9iwYfGGN7whvvWtb8X9998fX/rSl7Ier4WNGzfG6NGjsx4DkqTJpdPk0mhy1yiXLmsyHJkmly71Jkfocqk0uetoMhyZJpdOk0uXepMj0u9yuTQ5QpdTVLELtBEvnelx4MCBmDt3buzbty/q6upi5cqVzfcFp23f/e53Y//+/fHv//7v8e///u8t3vubv/mbWLhwYUaT/Y/PfvazsWTJkrjiiivihRdeiDPOOCPuuOOOOO6447IerSz07t07VqxYEUuWLInLL788du3aFWPHjo3bbrut1W0ZsnLppZdG//79Y+nSpbFjx44YNWpUfO5zn4uzzjor69FaePrpp2Pw4MFZjwHJ0uTSaHLlK4cmR5RHlzUZ2qbJpSmHJkfocik0uetoMrRNk0ujyflQDl0uhyZH6HKKehWLxWLWQwAAAAAAAADkQcU+gxYAAAAAAAAgNRZoAQAAAAAAAHqIBVoAAAAAAACAHmKBFgAAAAAAAKCHWKAFAAAAAAAA6CEWaAEAAAAAAAB6iAVaAAAAAAAAgB5igRYAAAAAAACgh1igBQAAAAAAAOghFmgBAAAAAAAAeogFWgAAAAAAAIAeYoEWAAAAAAAAoIf8//ZzK3iYTdb3AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 2400x400 with 8 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(ncols=4, figsize=(24, 4))\n",
    "for i, at in enumerate(attn[0:4]):\n",
    "    sns.heatmap(attn[i], ax=ax[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multi Head Attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='./images/attention_picture.png'>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's construct an MHA layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "    ''' Multi-Head Attention module '''\n",
    "\n",
    "    def __init__(self, n_head, d_model, d_k, d_v, dropout=0.1):\n",
    "        super().__init__()\n",
    "\n",
    "        self.n_head = n_head\n",
    "        self.d_k = d_k\n",
    "        self.d_v = d_v\n",
    "        self.d_model = d_model\n",
    "\n",
    "        self.w_qs = nn.Linear(d_model, n_head * d_k)\n",
    "        self.w_ks = nn.Linear(d_model, n_head * d_k)\n",
    "        self.w_vs = nn.Linear(d_model, n_head * d_v)\n",
    "\n",
    "        self.attention = ScaledDotProductAttention(\n",
    "            temperature=d_k**0.5) \n",
    "        self.layer_norm = nn.LayerNorm(d_model)\n",
    "\n",
    "        self.fc = nn.Linear(n_head * d_v, d_model)\n",
    "        nn.init.xavier_normal_(self.fc.weight)\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "        self.reset_parameters()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "         # normal distribution initialization better than kaiming(default in pytorch)\n",
    "        nn.init.normal_(self.w_qs.weight, mean=0,\n",
    "                        std=np.sqrt(2.0 / (self.d_model + self.d_k)))\n",
    "        nn.init.normal_(self.w_ks.weight, mean=0,\n",
    "                        std=np.sqrt(2.0 / (self.d_model + self.d_k)))\n",
    "        nn.init.normal_(self.w_vs.weight, mean=0,\n",
    "                        std=np.sqrt(2.0 / (self.d_model + self.d_v))) \n",
    "        \n",
    "    def forward(self, q, k, v, mask=None):\n",
    "        d_k, d_v, n_head = self.d_k, self.d_v, self.n_head\n",
    "\n",
    "        sz_b, len_q, _ = q.size()\n",
    "        sz_b, len_k, _ = k.size()\n",
    "        sz_b, len_v, _ = v.size()\n",
    "\n",
    "        residual = q\n",
    "\n",
    "        q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)\n",
    "        k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)\n",
    "        v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)\n",
    "\n",
    "        q = q.permute(2, 0, 1, 3).contiguous().view(-1, len_q, d_k)  # (n*b) x lq x dk\n",
    "        k = k.permute(2, 0, 1, 3).contiguous().view(-1, len_k, d_k)  # (n*b) x lk x dk\n",
    "        v = v.permute(2, 0, 1, 3).contiguous().view(-1, len_v, d_v)  # (n*b) x lv x dv\n",
    "        \n",
    "        if mask is not None:\n",
    "            mask = mask.repeat(n_head, 1, 1)  # (n*b) x .. x ..\n",
    "        output, attn = self.attention(q, k, v, mask=mask)\n",
    "\n",
    "        output = output.view(n_head, sz_b, len_q, d_v)\n",
    "        output = output.permute(1, 2, 0, 3).contiguous().view(sz_b, len_q, -1)  # b x lq x (n*dv)\n",
    "\n",
    "        output = self.dropout(self.fc(output))\n",
    "        output = self.layer_norm(output + residual)\n",
    "\n",
    "        return output, attn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Realization from Andrey Karpathy- https://github.com/karpathy/minGPT/blob/master/mingpt/model.py\n",
    "- Flash Attention - https://arxiv.org/abs/2205.14135"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Positionwise Feed Forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionwiseFeedForward(nn.Module):\n",
    "    ''' A two-feed-forward-layer module '''\n",
    "\n",
    "    def __init__(self, d_in, d_hid, dropout=0.1):\n",
    "        super().__init__()\n",
    "\n",
    "        # Use Conv1D\n",
    "        # position-wise\n",
    "        self.w_1 = nn.Conv1d(\n",
    "            d_in, d_hid, kernel_size=model_config.fft_conv1d_kernel[0], padding=model_config.fft_conv1d_padding[0])\n",
    "        # position-wise\n",
    "        self.w_2 = nn.Conv1d(\n",
    "            d_hid, d_in, kernel_size=model_config.fft_conv1d_kernel[1], padding=model_config.fft_conv1d_padding[1])\n",
    "\n",
    "        self.layer_norm = nn.LayerNorm(d_in)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "        output = x.transpose(1, 2)\n",
    "        output = self.w_2(F.relu(self.w_1(output)))\n",
    "        output = output.transpose(1, 2)\n",
    "        output = self.dropout(output)\n",
    "        output = self.layer_norm(output + residual)\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FFTBlock"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='./images/fft.png'>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Совмещаем все вместе в один FFT(Feed Forward Transformer) BLock. Теперь можно стакать эти слои"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FFTBlock(torch.nn.Module):\n",
    "    \"\"\"FFT Block\"\"\"\n",
    "\n",
    "    def __init__(self,\n",
    "                 d_model,\n",
    "                 d_inner,\n",
    "                 n_head,\n",
    "                 d_k,\n",
    "                 d_v,\n",
    "                 dropout=0.1):\n",
    "        super(FFTBlock, self).__init__()\n",
    "        self.slf_attn = MultiHeadAttention(\n",
    "            n_head, d_model, d_k, d_v, dropout=dropout)\n",
    "        self.pos_ffn = PositionwiseFeedForward(\n",
    "            d_model, d_inner, dropout=dropout)\n",
    "\n",
    "    def forward(self, enc_input, non_pad_mask=None, slf_attn_mask=None):\n",
    "        enc_output, enc_slf_attn = self.slf_attn(\n",
    "            enc_input, enc_input, enc_input, mask=slf_attn_mask)\n",
    "        \n",
    "        if non_pad_mask is not None:\n",
    "            enc_output *= non_pad_mask\n",
    "\n",
    "        enc_output = self.pos_ffn(enc_output)\n",
    "        \n",
    "        if non_pad_mask is not None:\n",
    "            enc_output *= non_pad_mask\n",
    "\n",
    "        return enc_output, enc_slf_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size = 16\n",
    "intermediate_size = 64\n",
    "n_head = 4\n",
    "batch_size = 4\n",
    "seq_len = 12\n",
    "\n",
    "fft_block = FFTBlock(hidden_size, intermediate_size, n_head, hidden_size // n_head, hidden_size // n_head)\n",
    "\n",
    "inp_tensor = torch.rand(batch_size, seq_len, hidden_size, dtype=torch.float32)\n",
    "\n",
    "out_tensor = fft_block(inp_tensor)[0]\n",
    "\n",
    "assert inp_tensor.shape == out_tensor.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Training tips - https://arxiv.org/pdf/1804.00247.pdf\n",
    "- Transformer without Tears - https://tnq177.github.io/data/transformers_without_tears.pdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Length Regulator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Aligner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_alignment(base_mat, duration_predictor_output):\n",
    "    N, L = duration_predictor_output.shape\n",
    "    for i in range(N):\n",
    "        count = 0\n",
    "        for j in range(L):\n",
    "            for k in range(duration_predictor_output[i][j]):\n",
    "                base_mat[i][count+k][j] = 1\n",
    "            count = count + duration_predictor_output[i][j]\n",
    "    return base_mat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Doubling each mel time frame is done via this function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.],\n",
       "        [0., 0., 1.],\n",
       "        [0., 0., 1.]]], dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "create_alignment(\n",
    "    torch.zeros(1, 6, 3).numpy(),\n",
    "    torch.LongTensor([[1,2,3]])\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Duration Predictor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='./images/duration_predictor.png'>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transpose(nn.Module):\n",
    "    def __init__(self, dim_1, dim_2):\n",
    "        super().__init__()\n",
    "        self.dim_1 = dim_1\n",
    "        self.dim_2 = dim_2\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x.transpose(self.dim_1, self.dim_2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Duration predictor is essentially a very simple fully-connected network trained on the regression problem (predicting the length of a phoneme, data is given from, say, Tacotron)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DurationPredictor(nn.Module):\n",
    "    \"\"\" Duration Predictor \"\"\"\n",
    "\n",
    "    def __init__(self, model_config: FastSpeechConfig):\n",
    "        super(DurationPredictor, self).__init__()\n",
    "\n",
    "        self.input_size = model_config.encoder_dim\n",
    "        self.filter_size = model_config.duration_predictor_filter_size\n",
    "        self.kernel = model_config.duration_predictor_kernel_size\n",
    "        self.conv_output_size = model_config.duration_predictor_filter_size\n",
    "        self.dropout = model_config.dropout\n",
    "\n",
    "        self.conv_net = nn.Sequential(\n",
    "            Transpose(-1, -2),\n",
    "            nn.Conv1d(\n",
    "                self.input_size, self.filter_size,\n",
    "                kernel_size=self.kernel, padding=1\n",
    "            ),\n",
    "            Transpose(-1, -2),\n",
    "            nn.LayerNorm(self.filter_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(self.dropout),\n",
    "            Transpose(-1, -2),\n",
    "            nn.Conv1d(\n",
    "                self.filter_size, self.filter_size,\n",
    "                kernel_size=self.kernel, padding=1\n",
    "            ),\n",
    "            Transpose(-1, -2),\n",
    "            nn.LayerNorm(self.filter_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(self.dropout)\n",
    "        )\n",
    "\n",
    "        self.linear_layer = nn.Linear(self.conv_output_size, 1)\n",
    "        self.relu = nn.ReLU()\n",
    "\n",
    "    def forward(self, encoder_output):\n",
    "        encoder_output = self.conv_net(encoder_output)\n",
    "            \n",
    "        out = self.linear_layer(encoder_output)\n",
    "        out = self.relu(out)\n",
    "        out = out.squeeze()\n",
    "        if not self.training:\n",
    "            out = out.unsqueeze(0)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "dur_predictor = DurationPredictor(model_config)\n",
    "\n",
    "inp_tensor = torch.rand(\n",
    "    2, # batch_size\n",
    "    12, #seq_len\n",
    "    model_config.encoder_dim,\n",
    "    dtype=torch.float32\n",
    ")\n",
    "\n",
    "dur_prediction = dur_predictor(inp_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.8276, 0.0733, 0.0547, 0.0000, 0.5168, 0.1581, 0.3389, 0.1272,\n",
       "         0.0000, 0.7756, 0.2618],\n",
       "        [0.2470, 0.1842, 0.3676, 0.2624, 0.1571, 0.3462, 0.4176, 0.2562, 0.0000,\n",
       "         0.3322, 0.2456, 0.5668]], grad_fn=<SqueezeBackward0>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dur_prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LengthRegulator(nn.Module):\n",
    "    \"\"\" Length Regulator \"\"\"\n",
    "\n",
    "    def __init__(self, model_config):\n",
    "        super(LengthRegulator, self).__init__()\n",
    "        self.duration_predictor = DurationPredictor(model_config)\n",
    "\n",
    "    def LR(self, x, duration_predictor_output, mel_max_length=None):\n",
    "        expand_max_len = torch.max(\n",
    "            torch.sum(duration_predictor_output, -1), -1)[0]\n",
    "        alignment = torch.zeros(duration_predictor_output.size(0),\n",
    "                                expand_max_len,\n",
    "                                duration_predictor_output.size(1)).numpy()\n",
    "        alignment = create_alignment(alignment,\n",
    "                                     duration_predictor_output.cpu().numpy())\n",
    "        alignment = torch.from_numpy(alignment).to(x.device)\n",
    "\n",
    "        output = alignment @ x\n",
    "        if mel_max_length:\n",
    "            output = F.pad(\n",
    "                output, (0, 0, 0, mel_max_length-output.size(1), 0, 0))\n",
    "        return output\n",
    "\n",
    "    def forward(self, x, alpha=1.0, target=None, mel_max_length=None):\n",
    "        duration_predictor_output = self.duration_predictor(x)\n",
    "\n",
    "        if target is not None:\n",
    "            output = self.LR(x, target, mel_max_length=mel_max_length)\n",
    "            return output, duration_predictor_output\n",
    "        else:\n",
    "            duration_predictor_output = (\n",
    "                (duration_predictor_output + 0.5) * alpha).int()\n",
    "            output = self.LR(x, duration_predictor_output)\n",
    "            mel_pos = torch.stack(\n",
    "                [torch.Tensor([i+1 for i in range(output.size(1))])]).long().to(device)\n",
    "\n",
    "            return output, mel_pos        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we build the whole model on the high level."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final BLock"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attention masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_non_pad_mask(seq):\n",
    "    assert seq.dim() == 2\n",
    "    return seq.ne(model_config.PAD).type(torch.float).unsqueeze(-1)\n",
    "\n",
    "def get_attn_key_pad_mask(seq_k, seq_q):\n",
    "    ''' For masking out the padding part of key sequence. '''\n",
    "    # Expand to fit the shape of key query attention matrix.\n",
    "    len_q = seq_q.size(1)\n",
    "    padding_mask = seq_k.eq(model_config.PAD)\n",
    "    padding_mask = padding_mask.unsqueeze(\n",
    "        1).expand(-1, len_q, -1)  # b x lq x lk\n",
    "\n",
    "    return padding_mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, model_config):\n",
    "        super(Encoder, self).__init__()\n",
    "        \n",
    "        len_max_seq=model_config.max_seq_len\n",
    "        n_position = len_max_seq + 1\n",
    "        n_layers = model_config.encoder_n_layer\n",
    "\n",
    "        self.src_word_emb = nn.Embedding(\n",
    "            model_config.vocab_size,\n",
    "            model_config.encoder_dim,\n",
    "            padding_idx=model_config.PAD\n",
    "        )\n",
    "\n",
    "        self.position_enc = nn.Embedding(\n",
    "            n_position,\n",
    "            model_config.encoder_dim,\n",
    "            padding_idx=model_config.PAD\n",
    "        )\n",
    "\n",
    "        self.layer_stack = nn.ModuleList([FFTBlock(\n",
    "            model_config.encoder_dim,\n",
    "            model_config.encoder_conv1d_filter_size,\n",
    "            model_config.encoder_head,\n",
    "            model_config.encoder_dim // model_config.encoder_head,\n",
    "            model_config.encoder_dim // model_config.encoder_head,\n",
    "            dropout=model_config.dropout\n",
    "        ) for _ in range(n_layers)])\n",
    "\n",
    "    def forward(self, src_seq, src_pos, return_attns=False):\n",
    "\n",
    "        enc_slf_attn_list = []\n",
    "\n",
    "        # -- Prepare masks\n",
    "        slf_attn_mask = get_attn_key_pad_mask(seq_k=src_seq, seq_q=src_seq)\n",
    "        non_pad_mask = get_non_pad_mask(src_seq)\n",
    "        \n",
    "        # -- Forward\n",
    "        enc_output = self.src_word_emb(src_seq) + self.position_enc(src_pos)\n",
    "\n",
    "        for enc_layer in self.layer_stack:\n",
    "            enc_output, enc_slf_attn = enc_layer(\n",
    "                enc_output,\n",
    "                non_pad_mask=non_pad_mask,\n",
    "                slf_attn_mask=slf_attn_mask)\n",
    "            if return_attns:\n",
    "                enc_slf_attn_list += [enc_slf_attn]\n",
    "        \n",
    "\n",
    "        return enc_output, non_pad_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    \"\"\" Decoder \"\"\"\n",
    "\n",
    "    def __init__(self, model_config):\n",
    "\n",
    "        super(Decoder, self).__init__()\n",
    "\n",
    "        len_max_seq=model_config.max_seq_len\n",
    "        n_position = len_max_seq + 1\n",
    "        n_layers = model_config.decoder_n_layer\n",
    "\n",
    "        self.position_enc = nn.Embedding(\n",
    "            n_position,\n",
    "            model_config.encoder_dim,\n",
    "            padding_idx=model_config.PAD,\n",
    "        )\n",
    "\n",
    "        self.layer_stack = nn.ModuleList([FFTBlock(\n",
    "            model_config.encoder_dim,\n",
    "            model_config.encoder_conv1d_filter_size,\n",
    "            model_config.encoder_head,\n",
    "            model_config.encoder_dim // model_config.encoder_head,\n",
    "            model_config.encoder_dim // model_config.encoder_head,\n",
    "            dropout=model_config.dropout\n",
    "        ) for _ in range(n_layers)])\n",
    "\n",
    "    def forward(self, enc_seq, enc_pos, return_attns=False):\n",
    "\n",
    "        dec_slf_attn_list = []\n",
    "\n",
    "        # -- Prepare masks\n",
    "        slf_attn_mask = get_attn_key_pad_mask(seq_k=enc_pos, seq_q=enc_pos)\n",
    "        non_pad_mask = get_non_pad_mask(enc_pos)\n",
    "\n",
    "        # -- Forward\n",
    "        dec_output = enc_seq + self.position_enc(enc_pos)\n",
    "\n",
    "        for dec_layer in self.layer_stack:\n",
    "            dec_output, dec_slf_attn = dec_layer(\n",
    "                dec_output,\n",
    "                non_pad_mask=non_pad_mask,\n",
    "                slf_attn_mask=slf_attn_mask)\n",
    "            if return_attns:\n",
    "                dec_slf_attn_list += [dec_slf_attn]\n",
    "\n",
    "        return dec_output\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mask_from_lengths(lengths, max_len=None):\n",
    "    if max_len == None:\n",
    "        max_len = torch.max(lengths).item()\n",
    "\n",
    "    ids = torch.arange(0, max_len, 1, device=lengths.device)\n",
    "    mask = (ids < lengths.unsqueeze(1)).bool()\n",
    "\n",
    "    return mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FastSpeech(nn.Module):\n",
    "    \"\"\" FastSpeech \"\"\"\n",
    "\n",
    "    def __init__(self, model_config):\n",
    "        super(FastSpeech, self).__init__()\n",
    "\n",
    "        self.encoder = Encoder(model_config)\n",
    "        self.length_regulator = LengthRegulator(model_config)\n",
    "        self.decoder = Decoder(model_config)\n",
    "\n",
    "        self.mel_linear = nn.Linear(model_config.decoder_dim, mel_config.num_mels)\n",
    "\n",
    "    def mask_tensor(self, mel_output, position, mel_max_length):\n",
    "        lengths = torch.max(position, -1)[0]\n",
    "        mask = ~get_mask_from_lengths(lengths, max_len=mel_max_length)\n",
    "        mask = mask.unsqueeze(-1).expand(-1, -1, mel_output.size(-1))\n",
    "        return mel_output.masked_fill(mask, 0.)\n",
    "\n",
    "    def forward(self, src_seq, src_pos, mel_pos=None, mel_max_length=None, length_target=None, alpha=1.0):\n",
    "       \n",
    "        encoder_output, _ = self.encoder(src_seq, src_pos)\n",
    "\n",
    "        if self.training:\n",
    "            length_regulator_output, duration_predictor_output = self.length_regulator(encoder_output,\n",
    "                                                                                       target=length_target,\n",
    "                                                                                       alpha=alpha,\n",
    "                                                                                       mel_max_length=mel_max_length)\n",
    "            decoder_output = self.decoder(length_regulator_output, mel_pos)\n",
    "\n",
    "            mel_output = self.mel_linear(decoder_output)\n",
    "            mel_output = self.mask_tensor(mel_output, mel_pos, mel_max_length)\n",
    "            residual = self.postnet(mel_output)\n",
    "            residual = self.last_linear(residual)\n",
    "            mel_postnet_output = mel_output + residual\n",
    "            mel_postnet_output = self.mask_tensor(mel_postnet_output,\n",
    "                                                  mel_pos,\n",
    "                                                  mel_max_length)\n",
    "\n",
    "            return mel_output, mel_postnet_output, duration_predictor_output\n",
    "        else:\n",
    "            length_regulator_output, decoder_pos = self.length_regulator(encoder_output,\n",
    "                                                                         alpha=alpha)\n",
    "\n",
    "            decoder_output = self.decoder(length_regulator_output, decoder_pos)\n",
    "\n",
    "            mel_output = self.mel_linear(decoder_output)\n",
    "            residual = self.postnet(mel_output)\n",
    "            residual = self.last_linear(residual)\n",
    "            mel_postnet_output = mel_output + residual\n",
    "\n",
    "            return mel_output, mel_postnet_output\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FastSpeechLoss(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.mse_loss = nn.MSELoss()\n",
    "        self.l1_loss = nn.L1Loss()\n",
    "\n",
    "    def forward(self, mel, duration_predicted, mel_target, duration_predictor_target):\n",
    "        mel_loss = self.mse_loss(mel, mel_target)\n",
    "\n",
    "        duration_predictor_loss = self.l1_loss(duration_predicted,\n",
    "                                               duration_predictor_target.float())\n",
    "\n",
    "        return mel_loss, duration_predictor_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "FastSpeech.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "toc-autonumbering": false,
  "toc-showcode": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}