diff --git a/base/AIVA_500m.ipynb b/base/AIVA_500m.ipynb index f633c80..bad4cb5 100644 --- a/base/AIVA_500m.ipynb +++ b/base/AIVA_500m.ipynb @@ -4,7 +4,9 @@ "metadata": { "colab": { "provenance": [], - "authorship_tag": "ABX9TyMgvNpTgHlkB38JeUIUdd7l", + "machine_shape": "hm", + "gpuType": "T4", + "authorship_tag": "ABX9TyOeYX5zp+reGmNxsWXca/e6", "include_colab_link": true }, "kernelspec": { @@ -13,7 +15,8 @@ }, "language_info": { "name": "python" - } + }, + "accelerator": "GPU" }, "cells": [ { @@ -23,16 +26,28 @@ "colab_type": "text" }, "source": [ - "\"Open" + "\"Open" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { - "id": "owjS3-sBcdBh" + "id": "owjS3-sBcdBh", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "7986e6e4-75e4-4f10-e868-dfbda7a0d3e7" }, - "outputs": [], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Mounted at /content/drive\n" + ] + } + ], "source": [ "from google.colab import drive\n", "drive.mount('/content/drive')" @@ -44,25 +59,59 @@ "!pip install tiktoken" ], "metadata": { - "id": "keNA2G8xfroc" + "id": "keNA2G8xfroc", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "f3204533-15b0-4b94-df77-93925fa224b1" }, - "execution_count": null, - "outputs": [] + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting tiktoken\n", + " Downloading tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n", + "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/1.8 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.3/1.8 MB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.8 MB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m20.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.12.25)\n", + "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2.31.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.6)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2024.2.2)\n", + "Installing collected packages: tiktoken\n", + "Successfully installed tiktoken-0.6.0\n" + ] + } + ] }, { "cell_type": "code", "source": [ "# data for model\n", - "with open('/content/drive/MyDrive/new_training_data.txt', 'r', encoding='utf-8') as file:\n", - " captions = file.read()\n", + "with open('/content/drive/MyDrive/training data/consolidated_350m.txt', 'r', encoding='utf-8') as file:\n", + " train_data = file.read()\n", "\n", - "print(len(captions)/1e6, 'million words')" + "print(len(train_data)/1e6, 'million words')" ], "metadata": { - "id": "BSh3yuTGfu21" + "id": "BSh3yuTGfu21", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "86e17e89-11a4-45cf-bfab-6f68002ef9bc" }, - "execution_count": null, - "outputs": [] + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "2274.16219 million words\n" + ] + } + ] }, { "cell_type": "code", @@ -71,16 +120,15 @@ "tokenizer = tiktoken.get_encoding(\"p50k_base\")\n", "tokenizer = tiktoken.encoding_for_model(\"text-davinci-003\")\n", "\n", - "input_data = tokenizer.encode(captions)\n", - "end_time = timeit.default_timer()\n", + "input_data = tokenizer.encode(train_data)\n", "\n", "print(\"total tokens\", len(input_data)/1e6, 'million')\n", - "print(f\"time taken to train the tokenizer {total_time}mins\")\n", "\n", "n = int(0.9*len(input_data)) # first 90% will be train, rest val\n", "train_data = input_data[:n]\n", "val_data = input_data[n:]\n", - "print(f\"train data {len(train_data) / 1e6} million'\\n'validation data {len(val_data) / 1e6} million\")" + "\n", + "del input_data, n" ], "metadata": { "id": "VmBZRVhqfyn2" @@ -97,243 +145,327 @@ "train_data = torch.tensor(train_data, dtype=torch.long)\n", "val_data = torch.tensor(val_data, dtype=torch.long)\n", "\n", + "print(f\"train data {(len(train_data) / 1e6):.0f} million\\nvalidation data {(len(val_data) / 1e6):.0f} million\")\n", "print(f\"train data = {train_data[:10]}, \\nval data = {val_data[:10]}\")" ], "metadata": { - "id": "rtfFsfgdf8rw" + "id": "rtfFsfgdf8rw", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "9aaaea88-25d5-4a97-ace1-7c817fca7270" }, - "execution_count": null, - "outputs": [] + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + ":4: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " train_data = torch.tensor(train_data, dtype=torch.long)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "train data 588 million\n", + "validation data 65 million\n", + "train data = tensor([ 3886, 25, 7443, 13, 785, 48073, 19433, 25, 2932, 860]), \n", + "val data = tensor([ 7579, 2885, 17941, 1847, 7446, 8696, 2389, 18310, 13, 3336])\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + ":5: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " val_data = torch.tensor(val_data, dtype=torch.long)\n" + ] + } + ] }, { "cell_type": "code", "source": [ - "import torch.nn as nn\n", - "from torch.nn import functional as F\n", - "\n", "# hyperparameters\n", - "batch_size = 32\n", - "block_size = 512\n", + "batch_size = 10\n", + "block_size = 256\n", "max_iters = 1000\n", "eval_interval = 100\n", - "learning_rate = 3e-4\n", + "learning_rate = 3e-5\n", "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", - "eval_iters = 50\n", - "d_embd = 512\n", - "n_head = 16\n", - "n_layer = 16\n", + "eval_iters = 10\n", + "d_model = 512\n", + "n_head = 20\n", + "n_layers = 18\n", "dropout = 0.2\n", - "norm_eps = 1e-05\n", - "# ------------\n", - "\n", - "torch.manual_seed(1400)\n", - "\n", - "# data loading\n", - "def get_batch(split):\n", - " # generate a small batch of data of inputs x and targets y\n", - " data = train_data if split == 'train' else val_data\n", - " ix = torch.randint(len(data) - block_size, (batch_size,))\n", - " x = torch.stack([data[i:i+block_size] for i in ix])\n", - " y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n", - " x, y = x.to(device), y.to(device)\n", - " return x, y\n", + "norm_eps = 1e-05" + ], + "metadata": { + "id": "tJuCsc1QPdts" + }, + "execution_count": 10, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "from torch.nn import functional as F\n", "\n", - "@torch.no_grad()\n", - "def estimate_loss():\n", - " out = {}\n", - " model.eval()\n", - " for split in ['train', 'val']:\n", - " losses = torch.zeros(eval_iters)\n", - " for k in range(eval_iters):\n", - " X, Y = get_batch(split)\n", - " logits, loss = model(X, Y)\n", - " losses[k] = loss.item()\n", - " out[split] = losses.mean()\n", - " model.train()\n", - " return out\n", + "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", "\n", - "class Head(nn.Module):\n", - " \"\"\" one head of self-attention \"\"\"\n", + "class RMSNorm(nn.Module):\n", + " def __init__(self, dim: int, eps: float = 1e-6):\n", + " \"\"\"\n", + " Initialize the RMSNorm normalization layer.\n", + " Args:\n", + " dim (int): The dimension of the input tensor.\n", + " eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6.\n", + " Attributes:\n", + " eps (float): A small value added to the denominator for numerical stability.\n", + " weight (nn.Parameter): Learnable scaling parameter.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.eps = eps\n", + " self.weight = nn.Parameter(torch.ones(dim))\n", + "\n", + " def _norm(self, x):\n", + " \"\"\"\n", + " Apply the RMSNorm normalization to the input tensor.\n", + " Args:\n", + " x (torch.Tensor): The input tensor.\n", + " Returns:\n", + " torch.Tensor: The normalized tensor.\n", + " \"\"\"\n", + " return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)\n", "\n", - " def __init__(self, d_embd, n_head, dropout, block_size):\n", - " head_size = d_embd // n_head\n", + " def forward(self, x):\n", + " \"\"\"\n", + " Forward pass through the RMSNorm layer.\n", + " Args:\n", + " x (torch.Tensor): The input tensor.\n", + " Returns:\n", + " torch.Tensor: The output tensor after applying RMSNorm.\n", + " \"\"\"\n", + " output = self._norm(x.float()).type_as(x)\n", + " return output * self.weight\n", + "\n", + "class UnMaskedHead(nn.Module):\n", + " def __init__(self, head_size, d_model, block_size, dropout):\n", " super().__init__()\n", - " self.key = nn.Linear(d_embd, head_size, bias=True)\n", - " self.query = nn.Linear(d_embd, head_size, bias=True)\n", - " self.value = nn.Linear(d_embd, head_size, bias=True)\n", - " self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n", + " self.key = nn.Linear(d_model, head_size, bias=True)\n", + " self.query = nn.Linear(d_model, head_size, bias=True)\n", + " self.value = nn.Linear(d_model, head_size, bias=True)\n", " self.dropout = nn.Dropout(dropout)\n", + " self.rel_pos_embd = nn.Parameter(torch.randn(block_size, block_size, head_size))\n", "\n", " def forward(self, x):\n", - " B,T,C = x.shape\n", - " key = self.key(x) # (B,T,hs)\n", - " query = self.query(x) # (B,T,hs)\n", - "\n", - " # compute attention scores (\"affinities\")\n", - " weights = query @ key.transpose(-2,-1) * key.shape[-1]**-0.5 # (B, T, hs) @ (B, hs, T) -> (B, T, T)\n", - " weights = weights.masked_fill(self.tril[:T, :T] == 0, float('-inf')) # (B, T, T)\n", - " weights = F.softmax(weights, dim=-1) # (B, T, T)\n", - " weights = self.dropout(weights)\n", - "\n", - " # perform the weighted aggregation of the values\n", - " value = self.value(x) # (B,T,hs)\n", - " out = weights @ value # (B, T, T) @ (B, T, hs) -> (B, T, hs)\n", - " return out\n", - "class MultiHeadAttention(nn.Module):\n", - " \"\"\" multiple heads of self-attention in parallel \"\"\"\n", + " B, T, C = x.shape\n", + " key = self.key(x)\n", + " query = self.query(x)\n", + "\n", + " scores = torch.matmul(query, key.transpose(-2, -1)) / (key.shape[-1] ** -0.5)\n", + " rel_pos_scores = torch.einsum('btc,tvc->btv', query, self.rel_pos_embd[:T, :T])\n", + " scores = scores + rel_pos_scores\n", + "\n", + " att_mat = F.softmax(scores, dim=-1)\n", + " att_mat = self.dropout(att_mat)\n", + " value = self.value(x)\n", + " output = torch.matmul(att_mat, value)\n", + " return output\n", "\n", - " def __init__(self, d_embd, n_head, dropout, block_size):\n", + "class UnMaskedAttention(nn.Module):\n", + " def __init__(self, d_model, block_size, dropout, n_head):\n", + " head_size = d_model // n_head\n", " super().__init__()\n", - " self.heads = nn.ModuleList([Head(d_embd=d_embd, n_head=n_head, dropout=dropout, block_size=block_size) for _ in range(n_head)])\n", - " self.proj = nn.Linear(n_head * (d_embd // n_head), d_embd)\n", + " self.heads = nn.ModuleList([UnMaskedHead(d_model=d_model, dropout=dropout, block_size=block_size, head_size=head_size) for _ in range(n_head)])\n", + " self.proj = nn.Linear(n_head * head_size, d_model)\n", " self.dropout = nn.Dropout(dropout)\n", "\n", " def forward(self, x):\n", " out = torch.cat([h(x) for h in self.heads], dim=-1)\n", - " out = self.dropout(out)\n", - "\n", + " out = self.dropout(self.proj(out))\n", " return out\n", "\n", - "class FeedForward:\n", - " \"\"\" dual linear layer with GELU function \"\"\"\n", - " def __init__(self, d_embd):\n", + "class MaskedHead(nn.Module):\n", + " def __init__(self, d_model, head_size, dropout, block_size):\n", " super().__init__()\n", - " self.fc1 = nn.Linear(d_embd, 4*d_embd) # n_ff = 4*d_embd\n", - " self.fc2 = nn.Linear(4*d_embd, d_embd) # n_ff = 4*d_embd\n", + " self.key = nn.Linear(d_model, head_size, bias=False)\n", + " self.query = nn.Linear(d_model, head_size, bias=False)\n", + " self.value = nn.Linear(d_model, head_size, bias=False)\n", + " self.dropout = nn.Dropout(dropout)\n", + " self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n", "\n", " def forward(self, x):\n", - " x = F.gelu(self.fc1(x)) # GELU insted of ReLU\n", - " x = self.fc2(x)\n", - " return x\n", + " B, T, C = x.shape\n", + " key = self.key(x)\n", + " query = self.query(x)\n", + "\n", + " scores = torch.matmul(query, key.transpose(-2, -1)) / (key.shape[-1] ** -0.5)\n", + " scores = scores.masked_fill(self.tril[:T, :T] == 0, float('-inf'))\n", "\n", - "class EncoderDecoderAttention(nn.Module):\n", - " \"\"\" separate attention layer for decoder layer \"\"\"\n", + " att_mat = F.softmax(scores, dim=-1)\n", + " att_mat = self.dropout(att_mat)\n", + " value = self.value(x)\n", + " output = torch.matmul(att_mat, value)\n", + " return output\n", "\n", - " def __init__(self, d_embd, n_head, dropout, block_size):\n", + "class CasualMaskedAttention(nn.Module):\n", + " def __init__(self, d_model, block_size, dropout, n_head):\n", + " head_size = d_model // n_head\n", " super().__init__()\n", - " self.heads = nn.ModuleList([Head(d_embd, n_head, dropout, block_size) for _ in range(n_head)])\n", - " self.proj = nn.Linear(n_head * (d_embd // n_head), d_embd)\n", + " self.heads = nn.ModuleList([MaskedHead(d_model=d_model, dropout=dropout, block_size=block_size, head_size=head_size) for _ in range(n_head)])\n", + " self.proj = nn.Linear(n_head * head_size, d_model)\n", " self.dropout = nn.Dropout(dropout)\n", "\n", - " def forward(self, query, key, value, mask=None):\n", - " x = torch.cat((key, query, value), dim=-1)\n", - " energies = []\n", - " for head in self.heads:\n", - " energy = head(x)\n", - " energies.append(energy.unsqueeze(1))\n", - " energy = torch.cat(energies, dim=1)\n", - " energy = self.proj(energy)\n", - " energy = self.dropout(energy)\n", + " def forward(self, x):\n", + " out = torch.cat([h(x) for h in self.heads], dim=-1)\n", + " out = self.dropout(self.proj(out))\n", + " return out\n", + "\n", + "class FinalHead(nn.Module):\n", + " def __init__(self, d_model, head_size, dropout, block_size):\n", + " super().__init__()\n", + " self.key = nn.Linear(d_model, head_size, bias=True)\n", + " self.query = nn.Linear(d_model, head_size, bias=True)\n", + " self.value = nn.Linear(d_model, head_size, bias=True)\n", + " self.dropout = nn.Dropout(dropout)\n", "\n", - " if mask is not None:\n", - " energy = energy.masked_fill(mask == 0, float('-inf'))\n", + " def forward(self, x, att):\n", + " B, T, C = x.shape\n", + " key = self.key(att)\n", + " query = self.query(att)\n", "\n", - " attention = F.softmax(energy, dim=-1)\n", - " output = torch.matmul(attention, value)\n", + " scores = torch.matmul(query, key.transpose(-2, -1)) / (key.shape[-1] ** -0.5)\n", "\n", + " att_mat = F.softmax(scores, dim=-1)\n", + " att_mat = self.dropout(att_mat)\n", + " value = self.value(x)\n", + " output = torch.matmul(att_mat, value)\n", " return output\n", "\n", - "class EncoderLayer(nn.Module):\n", - " \"\"\" Encoder Layer \"\"\"\n", - "\n", - " def __init__(self, d_embd, n_head, dropout, block_size):\n", + "class FinalAttention(nn.Module):\n", + " def __init__(self, d_model, block_size, dropout, n_head):\n", + " head_size = d_model // n_head\n", " super().__init__()\n", - " self.s_att = MultiHeadAttention(d_embd=d_embd, n_head=n_head, block_size=block_size, dropout=dropout)\n", - " self.ffwd = FeedForward(d_embd=d_embd)\n", + " self.heads = nn.ModuleList([FinalHead(d_model=d_model, dropout=dropout, block_size=block_size, head_size=head_size) for _ in range(n_head)])\n", + " self.proj = nn.Linear(n_head * head_size, d_model)\n", " self.dropout = nn.Dropout(dropout)\n", - " self.norm1 = nn.LayerNorm(d_embd)\n", - " self.norm2 = nn.LayerNorm(d_embd)\n", "\n", - " def forward(self, src, src_mask=None):\n", - " src2 = self.s_att(src)\n", - " src = src + self.dropout(src2)\n", - " src = self.norm1(src)\n", + " def forward(self, x, att):\n", + " out = torch.cat([h(x, att) for h in self.heads], dim=-1)\n", + " out = self.dropout(self.proj(out))\n", + " return out\n", + "\n", + "class FeedForward(nn.Module):\n", + " def __init__(self, d_model, dropout):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(d_model, 10*d_model),\n", + " nn.GELU(),\n", + " nn.Linear(10*d_model, d_model),\n", + " nn.Dropout(dropout)\n", + " )\n", + "\n", + " def forward(self, x):\n", + " return self.net(x)\n", "\n", - " src2 = self.ffwd(src)\n", - " src = src + self.dropout(src2)\n", - " src = self.norm2(src)\n", + "class EncoderNetwork(nn.Module):\n", + " def __init__(self, d_model, n_head, norm_eps, dropout, block_size):\n", + " super().__init__()\n", + " self.s_att = UnMaskedAttention(n_head=n_head, d_model=d_model, dropout=dropout, block_size=block_size)\n", + " self.ffwd = FeedForward(d_model, dropout)\n", + " self.dropout = nn.Dropout(dropout)\n", + " self.norm = RMSNorm(d_model, eps=norm_eps)\n", + "\n", + " def forward(self, src):\n", + " src_att = self.s_att(self.norm(src))\n", + " src_out = src + self.dropout(src_att)\n", "\n", - " return src\n", + " src = self.ffwd(self.norm(src_out))\n", + " src_f = src_out + self.dropout(src)\n", "\n", - "class DecoderLayer(nn.Module):\n", - " \"\"\" Decoder Layer \"\"\"\n", + " del src_att, src_out, src\n", + " return src_f\n", "\n", - " def __init__(self, d_embd, n_head, dropout, block_size) -> None:\n", + "class DecoderNetwork(nn.Module):\n", + " def __init__(self, d_model, n_head, norm_eps, dropout, block_size):\n", " super().__init__()\n", - " self.s_att = MultiHeadAttention(d_embd=d_embd, n_head=n_head, block_size=block_size, dropout=dropout)\n", - " self.enc_att = EncoderDecoderAttention(d_embd=d_embd, n_head=n_head, block_size=block_size, dropout=dropout)\n", - " self.ffwd = FeedForward(d_embd=d_embd)\n", + " self.m_att = CasualMaskedAttention(n_head=n_head, d_model=d_model, dropout=dropout, block_size=block_size)\n", + " self.f_att = FinalAttention(d_model=d_model, n_head=n_head, dropout=dropout, block_size=block_size)\n", + " self.ffwd = FeedForward(d_model, dropout)\n", " self.dropout = nn.Dropout(dropout)\n", - " self.norm1 = nn.LayerNorm(d_embd)\n", - " self.norm2 = nn.LayerNorm(d_embd)\n", - " self.norm3 = nn.LayerNorm(d_embd)\n", + " self.norm = RMSNorm(d_model, eps=norm_eps)\n", "\n", - " def forward(self, trg, enc_src, trg_mask=None, src_mask=None):\n", - " trg2 = self.s_att(trg)\n", - " trg = trg2 + self.dropout(trg2)\n", - " trg = self.norm1(trg)\n", + " def forward(self, src, att):\n", + " m_att_out = self.m_att(self.norm(src))\n", + " m_out = src + self.dropout(m_att_out)\n", "\n", - " trg2 = self.enc_att(trg, enc_src, enc_src)\n", - " trg = trg + self.dropout(trg2)\n", - " trg = self.norm2(trg)\n", + " f_out = self.f_att(self.norm(m_out), self.norm(att))\n", + " f_out = m_out + self.dropout(f_out)\n", "\n", - " trg2 = self.ffwd(trg)\n", - " trg = trg + self.dropout(trg2)\n", - " trg = self.norm3(trg)\n", + " src_f = self.ffwd(self.norm(f_out))\n", + " src_f = f_out + self.dropout(src_f)\n", "\n", - " return trg\n", + " del f_out, m_out, m_att_out, src, att\n", + " return src_f\n", "\n", "class Transformer(nn.Module):\n", - " def __init__(self):\n", + " def __init__(self, vocab_size):\n", " super().__init__()\n", - " self.d_embd = d_embd\n", " self.block_size = block_size\n", - "\n", - " self.token_embd = nn.Embedding(vocab_size, d_embd)\n", - " self.pos_embd = nn.Embedding(block_size, d_embd)\n", - " self.enc_layer = nn.ModuleList([EncoderLayer(n_head=n_head, block_size=block_size, dropout=dropout, d_embd=d_embd) for _ in range(n_layers)])\n", - " self.dec_layer = nn.ModuleList([DecoderLayer(n_head=n_head, block_size=block_size, dropout=dropout, d_embd=d_embd) for _ in range(n_layers)])\n", - "\n", - " self.norm_final = nn.LayerNorm(d_embd)\n", - " self.lm_head = nn.Linear(d_embd, vocab_size)\n", - " self.fc_out = nn.Linear(d_embd, vocab_size)\n", + " self.toked_model = nn.Embedding(vocab_size, d_model)\n", + " self.pos_encod = nn.Embedding(block_size, d_model)\n", + " self.enc_layer = nn.ModuleList([EncoderNetwork(n_head=n_head, norm_eps=norm_eps, block_size=block_size, dropout=dropout, d_model=d_model) for _ in range(n_layers)])\n", + " self.dec_layer = nn.ModuleList([DecoderNetwork(n_head=n_head, norm_eps=norm_eps, block_size=block_size, dropout=dropout, d_model=d_model) for _ in range(n_layers)])\n", + " self.norm_final = RMSNorm(d_model, eps=norm_eps)\n", + " self.linear_final = nn.Linear(d_model, vocab_size)\n", " self.dropout = nn.Dropout(dropout)\n", " self.apply(self._init_weights)\n", "\n", " def _init_weights(self, module):\n", + " \"\"\"\n", + " initialize weights of linear and embedding layers\n", + "\n", + " Args:\n", + " - module (nn.Module): the module to initialize weights for\n", + " \"\"\"\n", " if isinstance(module, nn.Linear):\n", " torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)\n", " if module.bias is not None:\n", - " torch.nn.init.zeros_(module.bias)\n", - " elif isinstance(module, nn.Embedding) and module.weight.numel() > 0:\n", + " torch.nn.init.zeros_(module.bias.data)\n", + " elif isinstance(module, nn.Embedding):\n", " torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)\n", "\n", - " def make_src_mask(self, src):\n", - " src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)\n", - " return src_mask\n", + " def forward(self, idx, targets=None):\n", + " \"\"\"\n", + " forward pass of the transformer model\n", "\n", - " def make_trg_mask(self, trg):\n", - " trg_pad_mask = (trg != self.trg_pad_idx).unsqueeze(1).unsqueeze(2)\n", - " trg_len = trg.shape[1]\n", - " trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len), device=trg.device)).bool()\n", - " trg_mask = trg_pad_mask & trg_sub_mask\n", - " return trg_mask\n", + " Args:\n", + " - idx (Tensor): input tensor representing token indices\n", + " - targets (Tensor): target tensor for computing loss during training\n", "\n", - " def forward(self, idx, targets=None):\n", + " Returns:\n", + " - logits (Tensor): output logits from the final linear layer\n", + " - loss (Tensor): optional. computed cross-entropy loss if targets are provided, else None\n", + " \"\"\"\n", " B, T = idx.shape\n", "\n", - " tok_embd = self.token_embd(idx)\n", - " pos_embd = self.pos_embd(torch.arange(T, device=device))\n", - " x = tok_embd + pos_embd\n", + " toked_model = self.toked_model(idx)\n", + " pos_encod = self.pos_encod(torch.arange(T, device=device))\n", + " x = toked_model + pos_encod\n", "\n", " for layer in self.enc_layer:\n", - " x = layer(x, None)\n", + " x_out = layer(x)\n", "\n", " for layer in self.dec_layer:\n", - " x = layer(x, x)\n", + " x_final = layer(x, x_out)\n", "\n", - " x = self.norm_final(x)\n", - " logits = self.lm_head(x)\n", + " x_final = self.norm_final(x_final)\n", + " logits = self.linear_final(x_final)\n", "\n", " if targets is None:\n", " loss = None\n", @@ -342,34 +474,156 @@ " B, T, C = logits.shape\n", " logits = logits.view(B*T, C)\n", " targets = targets.view(B*T)\n", - " loss = F.cross_entropy(logits, targets, ignore_index=-52, reduction='mean')\n", + " loss = F.cross_entropy(logits, targets)\n", "\n", " return logits, loss\n", "\n", - " def generate(self, idx, max_tokens=50):\n", - " for _ in range(max_tokens):\n", - " idx_cond = idx[:, -self.block_size: ]\n", - " logits, loss = self(idx_cond)\n", + " def generate(self, idx, max_new_tokens, temperature=1.0, top_k=0):\n", + " \"\"\"\n", + " generate new tokens using the trained model\n", + "\n", + " Args:\n", + " - idx (Tensor): input tensor representing initial token indices\n", + " - max_new_tokens (int): max no of new tokens to generate\n", + " - temperature (float): softmax temperature for sampling\n", + " - top_k (int): no of top tokens to consider in sampling\n", + "\n", + " Returns:\n", + " - generated_tokens (list): list of generated token indices\n", + " \"\"\"\n", + " generated_tokens = []\n", + "\n", + " for _ in range(max_new_tokens):\n", + " idx_cond = idx[:, -self.block_size:]\n", + " logits, _ = self(idx_cond)\n", " logits = logits[:, -1, :]\n", - " probs = F.softmax(logits, dim=-1)\n", - " idx_next = torch.multinomial(probs, num_samples=1)\n", - " idx = torch.cat((idx, idx_next), dim=1)\n", "\n", - " return idx, loss\n", + " scaled_logits = logits / temperature\n", + " if top_k > 0:\n", + " scaled_logits = self._top_k_filtering(scaled_logits, top_k)\n", + "\n", + " probs = F.softmax(scaled_logits, dim=-1)\n", + " sampled_idx = torch.multinomial(probs, num_samples=1)\n", + " generated_tokens.append(sampled_idx.item())\n", + " idx = torch.cat((idx, sampled_idx), dim=1)\n", + "\n", + " return generated_tokens\n", + "\n", + " def generate_masked_tokens(self, idx, masked_indices, temperature=1.0, top_k=0):\n", + " \"\"\"\n", + " Generate predictions for masked tokens using the trained model.\n", + "\n", + " Args:\n", + " - idx (Tensor): input tensor representing token indices\n", + " - masked_indices (Tensor): tensor of indices indicating masked positions\n", + " - temperature (float): softmax temperature for sampling\n", + " - top_k (int): no of top tokens to consider in sampling\n", + "\n", + " Returns:\n", + " - predicted_tokens (Tensor): tensor of predicted token indices\n", + " \"\"\"\n", + " B, T = idx.shape\n", + "\n", + " toked_model = self.toked_model(idx)\n", + " pos_encod = self.pos_encod(torch.arange(T, device=device))\n", + " x = toked_model + pos_encod\n", + "\n", + " for layer in self.enc_layer:\n", + " x_out = layer(x)\n", + "\n", + " for layer in self.dec_layer:\n", + " x_final = layer(x, x_out)\n", + "\n", + " x_masked = x_final.clone()\n", + " x_masked[masked_indices] = self.toked_model(torch.tensor([6], device=device))\n", + "\n", + " x_masked = self.norm_final(x_masked)\n", + " logits = self.linear_final(x_masked)\n", + "\n", + " masked_logits = logits[masked_indices].view(-1, logits.size(-1))\n", + " scaled_logits = masked_logits / temperature\n", + " if top_k > 0:\n", + " scaled_logits = self._top_k_filtering(scaled_logits, top_k)\n", + "\n", + " probs = F.softmax(scaled_logits, dim=-1)\n", + " predicted_indices = torch.argmax(probs, dim=-1)\n", "\n", + " return predicted_indices\n", "\n", - "model = Transformer()\n", - "# checkpoint_path = '/content/drive/MyDrive/52.9_transformer_model.pth'\n", - "# checkpoint = torch.load(checkpoint_path)\n", - "# model.load_state_dict(checkpoint)\n", + " def _top_k_filtering(self, logits, top_k):\n", + " \"\"\"\n", + " filter logits to keep only the top-k tokens\n", + "\n", + " Args:\n", + " - logits (Tensor): input tensor representing unscaled logits\n", + " - top_k (int): no of top tokens to keep\n", + "\n", + " Returns:\n", + " - filtered_logits (Tensor): filtered logits with only top-k tokens remaining\n", + " \"\"\"\n", + " values, indices = torch.topk(logits, top_k, dim=-1)\n", + " min_value = values[:, -1].unsqueeze(-1).expand_as(logits)\n", + " filtered_logits = torch.where(logits < min_value, torch.ones_like(logits) * -float('inf'), logits)\n", + "\n", + " return filtered_logits" + ], + "metadata": { + "id": "OusOJ_H8gARB" + }, + "execution_count": 11, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "\"\"\"\n", + " use this file to train the model\n", + "\n", + " working:\n", + " - imports vatious dependencies first, and then loads the training data\n", + " - tokenizes it, per-character basis\n", + " - loads the required hyper-parameters and the model file\n", + " - trains it till 'max_iters' and saves the model state, and generates outputs\n", + "\n", + " with the current set configuration, model can reach upto ~60million parameters\n", + " and can become ~99% accurate with next token prediction\n", + "\"\"\"\n", + "\n", + "torch.manual_seed(1400)\n", + "# data loading\n", + "def get_batch(split):\n", + " # generate a small batch of data of inputs x and targets y\n", + " data = train_data if split == 'train' else val_data\n", + " ix = torch.randint(len(data) - block_size, (batch_size,))\n", + " x = torch.stack([data[i:i+block_size] for i in ix])\n", + " y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n", + " x, y = x.to(device), y.to(device)\n", + " return x, y\n", + "\n", + "@torch.no_grad()\n", + "def estimate_loss():\n", + " out = {}\n", + " model.eval()\n", + " for split in ['train', 'val']:\n", + " losses = torch.zeros(eval_iters)\n", + " for k in range(eval_iters):\n", + " X, Y = get_batch(split)\n", + " logits, loss = model(X, Y)\n", + " losses[k] = loss.item()\n", + " out[split] = losses.mean()\n", + " model.train()\n", + " return out\n", + "\n", + "vocab_size = tokenizer.n_vocab\n", + "model = Transformer(vocab_size)\n", "m = model.to(device)\n", "\n", "# no of parameters\n", "n_param = sum(p.numel() for p in m.parameters())/1e6\n", - "print(n_param, 'million')\n", - "\n", - "# optimizer\n", + "print(f\"vocab size: {vocab_size}\")\n", + "print(f\"{n_param:.0f} million parameters\")\n", "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n", + "\n", "steps = []\n", "train_losses = []\n", "val_losses = []\n", @@ -391,22 +645,46 @@ " optimizer.step()" ], "metadata": { - "id": "OusOJ_H8gARB" + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "dORKqYKmPmit", + "outputId": "62d9926c-a50f-427b-abcf-bb71ec82348e" }, - "execution_count": null, - "outputs": [] + "execution_count": 12, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "vocab size: 50281\n", + "886 million parameters\n", + "step 0: train loss 10.9287, val loss 10.9229\n", + "step 100: train loss 8.3812, val loss 9.0325\n", + "step 200: train loss 7.2081, val loss 8.0959\n", + "step 300: train loss 6.7217, val loss 7.8882\n", + "step 400: train loss 6.5446, val loss 7.8266\n", + "step 500: train loss 6.8072, val loss 7.8396\n", + "step 600: train loss 6.4265, val loss 7.6559\n", + "step 700: train loss 6.3871, val loss 7.7765\n", + "step 800: train loss 6.4383, val loss 7.5266\n", + "step 900: train loss 6.2296, val loss 7.3788\n", + "step 999: train loss 6.3129, val loss 7.3048\n" + ] + } + ] }, { "cell_type": "code", "source": [ - "# save the trained model\n", - "torch.save(model.state_dict(), f\"{n_param:.1f}_model_dict.pth\")\n", - "torch.save(model, f\"{n_param:.1f}_model.pth\")" + "model_save_name = f'aiva_base-{n_param:.0f}m.pth'\n", + "path = f\"/content/drive/MyDrive/{model_save_name}\"\n", + "torch.save(model.state_dict(), path)" ], "metadata": { "id": "e6NM24zMhH_2" }, - "execution_count": null, + "execution_count": 15, "outputs": [] }, { @@ -425,25 +703,63 @@ "plt.show()" ], "metadata": { - "id": "mmFhDw7KhK0t" + "id": "mmFhDw7KhK0t", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 564 + }, + "outputId": "d2bd4dcf-f197-447b-8f19-48aaa3f3c0d2" }, - "execution_count": null, - "outputs": [] + "execution_count": 17, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0oAAAIjCAYAAAA9VuvLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACDiUlEQVR4nOzdd3wUdf7H8dem90I6EEIJvUmXDtJFxI7oqVjPE09RT+88ez97PfvvwK7YEBWRKl0IvfcSSkIIpIe03fn9scmyCQGSkGQ2yfv5eOwjszOzs58lo+yb73c+YzEMw0BEREREREQc3MwuQERERERExNUoKImIiIiIiJShoCQiIiIiIlKGgpKIiIiIiEgZCkoiIiIiIiJlKCiJiIiIiIiUoaAkIiIiIiJShoKSiIiIiIhIGQpKIiIiIiIiZSgoiYiIiIiIlKGgJCLSAEybNg2LxcLq1avNLqVCli1bxuWXX05UVBTe3t40b96cv/71ryQmJppdWrn279/PzTffTKtWrfDx8SE6OppBgwbxxBNPlNrv3XffZdq0aeYUKSIilWIxDMMwuwgREalZ06ZN4+abbyYhIYGePXuaXc5Zvf3229x77720bNmSSZMmERMTw7Zt2/j4448BmDVrFv369TO5ylN2795Nr1698PX15ZZbbqF58+YkJSWxdu1afvvtN/Ly8hz7durUifDwcP744w/zChYRkQrxMLsAERGREsuWLWPKlCkMGDCA2bNn4+fn59j2t7/9jf79+3PVVVexZcsWQkNDa62unJwc/P39y932+uuvk52dzfr164mLiyu1LSUlpTbKExGRGqCpdyIi4rBu3TrGjBlDUFAQAQEBDBs2jD///LPUPoWFhTz11FO0bt0aHx8fwsLCGDBgAHPnznXsk5yczM0330zTpk3x9vYmJiaG8ePHs3///rO+/zPPPIPFYuGTTz4pFZIAWrVqxUsvvURSUhIffPABAK+88goWi4UDBw6cdqyHH34YLy8v0tLSHOtWrlzJ6NGjCQ4Oxs/Pj8GDB7Ns2bJSr3vyySexWCxs3bqV6667jtDQUAYMGHDGmvfs2UPTpk1PC0kAkZGRjuXmzZuzZcsWFi1ahMViwWKxMGTIEMf29PR0pkyZQmxsLN7e3sTHx/Piiy9is9kc++zfvx+LxcIrr7zC66+/TlxcHL6+vgwePJjNmzeXeu+q/g5ERMROI0oiIgLAli1bGDhwIEFBQTz00EN4enrywQcfMGTIEBYtWkSfPn0Ae5B44YUXuO222+jduzeZmZmsXr2atWvXMmLECACuvPJKtmzZwt///neaN29OSkoKc+fOJTExkebNm5f7/rm5ucyfP5+BAwfSokWLcveZMGECd9xxB7/88gv/+te/uOaaa3jooYeYPn06Dz74YKl9p0+fzsiRIx0jTwsWLGDMmDH06NGDJ554Ajc3N6ZOncpFF13EkiVL6N27d6nXX3311bRu3Zrnn3+es81Sj4uLY968eSxYsICLLrrojPu98cYb/P3vfycgIIBHHnkEgKioKMdnHzx4MIcPH+avf/0rzZo1Y/ny5Tz88MMkJSXxxhtvlDrWp59+SlZWFpMnTyYvL48333yTiy66iE2bNjmOWZXfgYiIODFERKTemzp1qgEYCQkJZ9znsssuM7y8vIw9e/Y41h05csQIDAw0Bg0a5FjXtWtXY+zYsWc8TlpamgEYL7/8cqVqXL9+vQEY995771n369Kli9GoUSPH8759+xo9evQotc+qVasMwPj0008NwzAMm81mtG7d2hg1apRhs9kc++Xm5hotWrQwRowY4Vj3xBNPGIAxceLECtW9efNmw9fX1wCMCy64wLj33nuNGTNmGDk5Oaft27FjR2Pw4MGnrX/mmWcMf39/Y+fOnaXW/+tf/zLc3d2NxMREwzAMY9++fQZg+Pr6GocOHXLst3LlSgMw7rvvPsMwqv47EBGRUzT1TkREsFqtzJkzh8suu4yWLVs61sfExHDdddexdOlSMjMzAQgJCWHLli3s2rWr3GP5+vri5eXFH3/8UWra27lkZWUBEBgYeNb9AgMDHbWAfZRpzZo17Nmzx7Hum2++wdvbm/HjxwOwfv16du3axXXXXcfx48dJTU0lNTWVnJwchg0bxuLFi0tNcQO48847K1R3x44dWb9+PX/5y1/Yv38/b775JpdddhlRUVF89NFHFTrGt99+y8CBAwkNDXXUlpqayvDhw7FarSxevLjU/pdddhlNmjRxPO/duzd9+vRh1qxZQNV/ByIicoqCkoiIcOzYMXJzc2nbtu1p29q3b4/NZuPgwYMAPP3006Snp9OmTRs6d+7Mgw8+yMaNGx37e3t78+KLL/Lbb78RFRXFoEGDeOmll0hOTj5rDSUBqSQwnUlWVlapMHX11Vfj5ubGN998A4BhGHz77beOa60AR6i76aabiIiIKPX4+OOPyc/PJyMjo9T7nGn6X3natGnDZ599RmpqKhs3buT555/Hw8ODO+64g3nz5p3z9bt27WL27Nmn1TZ8+HDg9KYQrVu3LreGkuuPqvo7EBGRUxSURESkUgYNGsSePXv43//+R6dOnfj444/p3r27o303wJQpU9i5cycvvPACPj4+PPbYY7Rv355169ad8bjx8fF4eHiUCl1l5efns2PHDjp06OBY17hxYwYOHMj06dMB+PPPP0lMTGTChAmOfUpGi15++WXmzp1b7iMgIKDUe/n6+lbuDwZwd3enc+fOPPzww/z4448AfPHFF+d8nc1mY8SIEWes7corr6x0LVX5HYiIyClq5iAiIkRERODn58eOHTtO27Z9+3bc3NyIjY11rGvUqBE333wzN998M9nZ2QwaNIgnn3yS2267zbFPq1ateOCBB3jggQfYtWsXF1xwAa+++iqff/55uTX4+/szdOhQFixYwIEDB8rtIjd9+nTy8/O55JJLSq2fMGECd911Fzt27OCbb77Bz8+PcePGlaoFICgoyDFKU9NK7leVlJTkWGexWMrdt1WrVmRnZ1e4tvKmPe7cufO0Jg2V/R2IiMgpGlESERHc3d0ZOXIkP/30U6n20UePHuXLL79kwIABjmlsx48fL/XagIAA4uPjyc/PB+wd3Jxvsgr2L+yBgYGOfc7k0UcfxTAMJk2axMmTJ0tt27dvHw899BAxMTH89a9/LbXtyiuvxN3dna+++opvv/2WSy65pNR9j3r06EGrVq145ZVXyM7OPu19jx07dta6zmbJkiUUFhaetr7keiHn6Yz+/v6kp6eftu8111zDihUr+P3330/blp6eTlFRUal1M2bM4PDhw47nq1atYuXKlYwZMwY4v9+BiIjYaURJRKQB+d///sfs2bNPW3/vvffy7LPPMnfuXAYMGMBdd92Fh4cHH3zwAfn5+bz00kuOfTt06MCQIUPo0aMHjRo1YvXq1Xz33XfcfffdgH1kY9iwYVxzzTV06NABDw8PfvzxR44ePcq111571voGDRrEK6+8wv3330+XLl2YNGkSMTExbN++nY8++gibzcasWbNOu9lsZGQkQ4cO5bXXXiMrK6vUtDsANzc3Pv74Y8aMGUPHjh25+eabadKkCYcPH2bhwoUEBQXx888/V+nP9MUXX2TNmjVcccUVdOnSBYC1a9fy6aef0qhRI6ZMmeLYt0ePHrz33ns8++yzxMfHExkZyUUXXcSDDz7IzJkzueSSS5g0aRI9evQgJyeHTZs28d1337F//37Cw8Mdx4mPj2fAgAH87W9/Iz8/nzfeeIOwsDAeeuih8/4diIhIMbPb7omISM0raQ9+psfBgwcNwzCMtWvXGqNGjTICAgIMPz8/Y+jQocby5ctLHevZZ581evfubYSEhBi+vr5Gu3btjOeee84oKCgwDMMwUlNTjcmTJxvt2rUz/P39jeDgYKNPnz7G9OnTK1zv4sWLjfHjxxvh4eGGp6en0axZM+P222839u/ff8bXfPTRRwZgBAYGGidPnix3n3Xr1hlXXHGFERYWZnh7extxcXHGNddcY8yfP9+xT0l78GPHjlWo1mXLlhmTJ082OnXqZAQHBzvqnTRpUqlW64ZhGMnJycbYsWONwMBAAyjVKjwrK8t4+OGHjfj4eMPLy8sIDw83+vXrZ7zyyiuOP9uS9uAvv/yy8eqrrxqxsbGGt7e3MXDgQGPDhg2OY1XH70BEpKGzGMZZ7qInIiIiLmP//v20aNGCl19+mX/84x9mlyMiUq/pGiUREREREZEyFJRERERERETKUFASEREREREpQ9coiYiIiIiIlKERJRERERERkTIUlERERERERMqo9zectdlsHDlyhMDAQCwWi9nliIiIiIiISQzDICsri8aNG+PmdvYxo3oflI4cOUJsbKzZZYiIiIiIiIs4ePAgTZs2Pes+9T4oBQYGAvY/jKCgIJOrERERERERs2RmZhIbG+vICGdT74NSyXS7oKAgBSUREREREanQJTlq5iAiIiIiIlKGgpKIiIiIiEgZCkoiIiIiIiJl1PtrlERERETE9VitVgoLC80uQ+oZd3d3PDw8quW2QApKIiIiIlKrsrOzOXToEIZhmF2K1EN+fn7ExMTg5eV1XsdRUBIRERGRWmO1Wjl06BB+fn5ERERUy7/8i4D9ZrIFBQUcO3aMffv20bp163PeVPZsFJREREREpNYUFhZiGAYRERH4+vqaXY7UM76+vnh6enLgwAEKCgrw8fGp8rFMbeawePFixo0bR+PGjbFYLMyYMaPU9h9++IGRI0cSFhaGxWJh/fr1ptQpIiIiItVLI0lSU85nFKnUcarlKFWUk5ND165d+e9//3vG7QMGDODFF1+s5cpERERERKQhM3Xq3ZgxYxgzZswZt99www0A7N+/v5YqEhERERERqYf3UcrPzyczM7PUQ0RERETE1TRv3pw33njD7DLkDOpdUHrhhRcIDg52PGJjY80uSURERETqMIvFctbHk08+WaXjJiQkcMcdd5xXbUOGDGHKlCnndQwpX73revfwww9z//33O55nZmYqLImIiIhIlSUlJTmWv/nmGx5//HF27NjhWBcQEOBYNgwDq9WKh8e5v2ZHRERUb6FSrerdiJK3tzdBQUGlHiIiIiLimgzDILegyJRHRW94Gx0d7XgEBwdjsVgcz7dv305gYCC//fYbPXr0wNvbm6VLl7Jnzx7Gjx9PVFQUAQEB9OrVi3nz5pU6btmpdxaLhY8//pjLL78cPz8/WrduzcyZM8/rz/f777+nY8eOeHt707x5c1599dVS2999911at26Nj48PUVFRXHXVVY5t3333HZ07d8bX15ewsDCGDx9OTk7OedVTl9S7ESURERERqTtOFlrp8Pjvprz31qdH4edVPV+H//Wvf/HKK6/QsmVLQkNDOXjwIBdffDHPPfcc3t7efPrpp4wbN44dO3bQrFmzMx7nqaee4qWXXuLll1/m7bff5vrrr+fAgQM0atSo0jWtWbOGa665hieffJIJEyawfPly7rrrLsLCwpg0aRKrV6/mnnvu4bPPPqNfv36cOHGCJUuWAPZRtIkTJ/LSSy9x+eWXk5WVxZIlSyocLusDU4NSdnY2u3fvdjzft28f69evp1GjRjRr1owTJ06QmJjIkSNHABxDnCUJXkRERETEFTz99NOMGDHC8bxRo0Z07drV8fyZZ57hxx9/ZObMmdx9991nPM6kSZOYOHEiAM8//zxvvfUWq1atYvTo0ZWu6bXXXmPYsGE89thjALRp04atW7fy8ssvM2nSJBITE/H39+eSSy4hMDCQuLg4unXrBtiDUlFREVdccQVxcXEAdO7cudI11GWmBqXVq1czdOhQx/OSa4tuuukmpk2bxsyZM7n55psd26+99loAnnjiiSpfNGem/ak5LNhyiOv7x+Pt4W52OSIiIiKm8/V0Z+vTo0x77+rSs2fPUs+zs7N58skn+fXXXx2h4+TJkyQmJp71OF26dHEs+/v7ExQUREpKSpVq2rZtG+PHjy+1rn///rzxxhtYrVZGjBhBXFwcLVu2ZPTo0YwePdox7a9r164MGzaMzp07M2rUKEaOHMlVV11FaGholWqpi0wNSkOGDDnr8N2kSZOYNGlS7RVUg2yFBax691Yusy5hbcgc+nbtaHZJIiIiIqazWCzVNv3NTP7+/qWe/+Mf/2Du3Lm88sorxMfH4+vry1VXXUVBQcFZj+Pp6VnqucViwWazVXu9AIGBgaxdu5Y//viDOXPm8Pjjj/Pkk0+SkJBASEgIc+fOZfny5cyZM4e3336bRx55hJUrV9KiRYsaqcfV1LtmDq7KzdOLPj4HaWTJJnfF/5ldjoiIiIjUoGXLljFp0iQuv/xyOnfuTHR0NPv376/VGtq3b8+yZctOq6tNmza4u9tH0zw8PBg+fDgvvfQSGzduZP/+/SxYsACwh7T+/fvz1FNPsW7dOry8vPjxxx9r9TOYqe7H9zokp+skWPEPOif/iK3wP7h5epldkoiIiIjUgNatW/PDDz8wbtw4LBYLjz32WI2NDB07doz169eXWhcTE8MDDzxAr169eOaZZ5gwYQIrVqzgnXfe4d133wXgl19+Ye/evQwaNIjQ0FBmzZqFzWajbdu2rFy5kvnz5zNy5EgiIyNZuXIlx44do3379jXyGVyRRpRqUavB13PcCCKSEySu+NbsckRERESkhrz22muEhobSr18/xo0bx6hRo+jevXuNvNeXX35Jt27dSj0++ugjunfvzvTp0/n666/p1KkTjz/+OE8//bTj0paQkBB++OEHLrroItq3b8/777/PV199RceOHQkKCmLx4sVcfPHFtGnThkcffZRXX32VMWPG1MhncEUWo573+MvMzCQ4OJiMjAyXuKfS7DfvYnTaFyQG9aDZ/QvMLkdERESkVuXl5bFv3z5atGiBj4+P2eVIPXS2c6wy2UAjSrWt5ySshoVmmWsgZZvZ1YiIiIiISDkUlGpZ327dmG/0ACBzyfsmVyMiIiIiIuVRUKplwX6erI64AgCfrd9CfpbJFYmIiIiISFkKSiaI6TaaPbYYvKw5sOFrs8sREREREZEyFJRMMKJjDJ9bhwNQtPIjqN/9NERERERE6hwFJRM0DfVjc/hYcg1vPI7vgP1LzS5JREREREScKCiZpF+nVsyw9rc/SfjI3GJERERERKQUBSWTjOwYxafWkQAY236BzCMmVyQiIiIiIiUUlEzSISaIrOC2rLK1xWJYYc0nZpckIiIiIiLFFJRMYrFYGNEhis+KRthXrJkG1kJTaxIRERGRmjNkyBCmTJnieN68eXPeeOONs77GYrEwY8aM837v6jpOQ6KgZKIRHaKYbetNKiGQnQzbfja7JBEREREpY9y4cYwePbrcbUuWLMFisbBx48ZKHzchIYE77rjjfMsr5cknn+SCCy44bX1SUhJjxoyp1vcqa9q0aYSEhNToe9QmBSUT9W7RCF8fH74sGmJfkfCxqfWIiIiIyOluvfVW5s6dy6FDh07bNnXqVHr27EmXLl0qfdyIiAj8/Pyqo8Rzio6Oxtvbu1beq75QUDKRp7sbF7WL5MuiYdhwhwPL4OgWs8sSERERqT2GAQU55jwqeC/LSy65hIiICKZNm1ZqfXZ2Nt9++y233norx48fZ+LEiTRp0gQ/Pz86d+7MV199ddbjlp16t2vXLgYNGoSPjw8dOnRg7ty5p73mn//8J23atMHPz4+WLVvy2GOPUVhov3xj2rRpPPXUU2zYsAGLxYLFYnHUXHbq3aZNm7jooovw9fUlLCyMO+64g+zsbMf2SZMmcdlll/HKK68QExNDWFgYkydPdrxXVSQmJjJ+/HgCAgIICgrimmuu4ejRo47tGzZsYOjQoQQGBhIUFESPHj1YvXo1AAcOHGDcuHGEhobi7+9Px44dmTVrVpVrqQiPGj26nNOIDtHMWH+Epe69GWRdYR9VuuR1s8sSERERqR2FufB8Y3Pe+99HwMv/nLt5eHhw4403Mm3aNB555BEsFgsA3377LVarlYkTJ5KdnU2PHj345z//SVBQEL/++is33HADrVq1onfv3ud8D5vNxhVXXEFUVBQrV64kIyOj1PVMJQIDA5k2bRqNGzdm06ZN3H777QQGBvLQQw8xYcIENm/ezOzZs5k3bx4AwcHBpx0jJyeHUaNG0bdvXxISEkhJSeG2227j7rvvLhUGFy5cSExMDAsXLmT37t1MmDCBCy64gNtvv/2cn6e8z1cSkhYtWkRRURGTJ09mwoQJ/PHHHwBcf/31dOvWjffeew93d3fWr1+Pp6cnAJMnT6agoIDFixfj7+/P1q1bCQgIqHQdlaGgZLLBbSPwcnfjvZMXMchrBWz4BoY/CT6nn9QiIiIiYo5bbrmFl19+mUWLFjFkyBDAPu3uyiuvJDg4mODgYP7xj3849v/73//O77//zvTp0ysUlObNm8f27dv5/fffadzYHhyff/75064revTRRx3LzZs35x//+Adff/01Dz30EL6+vgQEBODh4UF0dPQZ3+vLL78kLy+PTz/9FH9/e1B85513GDduHC+++CJRUVEAhIaG8s477+Du7k67du0YO3Ys8+fPr1JQmj9/Pps2bWLfvn3ExsYC8Omnn9KxY0cSEhLo1asXiYmJPPjgg7Rr1w6A1q1bO16fmJjIlVdeSefOnQFo2bJlpWuoLAUlkwV4e9AvPow/dnTghF8LGuXus4elPtV7YZ+IiIiIS/L0s4/smPXeFdSuXTv69evH//73P4YMGcLu3btZsmQJTz/9NABWq5Xnn3+e6dOnc/jwYQoKCsjPz6/wNUjbtm0jNjbWEZIA+vbte9p+33zzDW+99RZ79uwhOzuboqIigoKCKvw5St6ra9eujpAE0L9/f2w2Gzt27HAEpY4dO+Lu7u7YJyYmhk2bNlXqvZzfMzY21hGSADp06EBISAjbtm2jV69e3H///dx222189tlnDB8+nKuvvppWrVoBcM899/C3v/2NOXPmMHz4cK688soqXRdWGbpGyQWM6BAFWPjOrbibSsLHFZ4zKyIiIlKnWSz26W9mPIqn0FXUrbfeyvfff09WVhZTp06lVatWDB48GICXX36ZN998k3/+858sXLiQ9evXM2rUKAoKCqrtj2rFihVcf/31XHzxxfzyyy+sW7eORx55pFrfw1nJtLcSFosFm81WI+8F9o59W7ZsYezYsSxYsIAOHTrw448/AnDbbbexd+9ebrjhBjZt2kTPnj15++23a6wWUFByCcPb21P726k9sHn6QeoO2LfY5KpERERExNk111yDm5sbX375JZ9++im33HKL43qlZcuWMX78eP7yl7/QtWtXWrZsyc6dOyt87Pbt23Pw4EGSkpIc6/78889S+yxfvpy4uDgeeeQRevbsSevWrTlw4ECpfby8vLBared8rw0bNpCTk+NYt2zZMtzc3Gjbtm2Fa66Mks938OBBx7qtW7eSnp5Ohw4dHOvatGnDfffdx5w5c7jiiiuYOnWqY1tsbCx33nknP/zwAw888AAfffRRjdRaQkHJBUQF+XBBbAhZ+LEn+hL7yoSa/cWLiIiISOUEBAQwYcIEHn74YZKSkpg0aZJjW+vWrZk7dy7Lly9n27Zt/PWvfy3V0e1chg8fTps2bbjpppvYsGEDS5Ys4ZFHHim1T+vWrUlMTOTrr79mz549vPXWW44RlxLNmzdn3759rF+/ntTUVPLz8097r+uvvx4fHx9uuukmNm/ezMKFC/n73//ODTfc4Jh2V1VWq5X169eXemzbto3hw4fTuXNnrr/+etauXcuqVau48cYbGTx4MD179uTkyZPcfffd/PHHHxw4cIBly5aRkJBA+/btAZgyZQq///47+/btY+3atSxcuNCxraYoKLkI+/Q7mFY03L5i+yzIOGxiRSIiIiJS1q233kpaWhqjRo0qdT3Ro48+Svfu3Rk1ahRDhgwhOjqayy67rMLHdXNz48cff+TkyZP07t2b2267jeeee67UPpdeein33Xcfd999NxdccAHLly/nscceK7XPlVdeyejRoxk6dCgRERHltij38/Pj999/58SJE/Tq1YurrrqKYcOG8c4771TuD6Mc2dnZdOvWrdRj3LhxWCwWfvrpJ0JDQxk0aBDDhw+nZcuWfPPNNwC4u7tz/PhxbrzxRtq0acM111zDmDFjeOqppwB7AJs8eTLt27dn9OjRtGnThnffffe86z0bi2HU74thMjMzCQ4OJiMjo9IXutWmXUezGPH6Yrzc3djW8i3cDy6HQQ/CRY+e+8UiIiIidUReXh779u2jRYsW+Pj4mF2O1ENnO8cqkw00ouQi4iMDaBHuT4HVxobGV9tXrvkEimrm4jwRERERETkzBSUXYbFYHNPvPk/vDAHRkJMC22aaXJmIiIiISMOjoORCSoLSvB0nsHa/yb4y4WMTKxIRERERaZgUlFxI92ahhPl7kZlXxLrwS8HiDokrIHmz2aWJiIiIiDQoCkouxN3NwrD2kQD8st8C7dUqXEREROqnet5PTExUXeeWgpKLGdEhGoC5W49i9LrNvnLjdDiZbl5RIiIiItXE3d0dgIICNaySmpGbmwuAp6fneR3HozqKkeozID4cH083DqefZKtXfzpGtIdj22DDV3Dh38wuT0REROS8eHh44Ofnx7Fjx/D09MTNTf9uL9XDMAxyc3NJSUkhJCTEEcqrSkHJxfh6uTOodQRzth5l7rYUOva6FWb9w97Uoc+dYLGYXaKIiIhIlVksFmJiYti3bx8HDhwwuxyph0JCQoiOjj7v4ygouaARHaKYs/Uoc7YcZcqd18K8p+D4btj7B7QaanZ5IiIiIufFy8uL1q1ba/qdVDtPT8/zHkkqoaDkgoa1j8LNAluTMjmU607TrtfaGzokfKygJCIiIvWCm5sbPj4+ZpchckaaFOqCGvl70bN5IwDmbT0KJU0ddsyC9IMmViYiIiIi0jAoKLmokcU3n52z9ShEtoPmA8GwwZqpJlcmIiIiIlL/KSi5qBHFQWnlvhNk5BaeGlVa8wkU5ZtYmYiIiIhI/aeg5KLiwvxpExWA1WawcEcKtBsLgTGQmwpbfzK7PBERERGRek1ByYWNdLr5LO6e0ONm+4aEj02sSkRERESk/lNQcmEl0+/+2JFCfpEVetwEbh5wcCUkbTS5OhERERGR+ktByYV1bhJMVJA3OQVWlu85DoHR0P5S+8aEj8wtTkRERESkHlNQcmFubhbHqNLcrUftK3vfbv+58Vs4mWZSZSIiIiIi9ZuCkosb4XSdks1mQLO+ENkBik7C+i9Nrk5EREREpH5SUHJxF7ZsRIC3B8ey8tlwKB0sllOtwhM+BpvN1PpEREREROojBSUX5+3hzuC2EYDT9LsuE8A7CE7shb0LTKxORERERKR+UlCqA0YWX6c0pyQoeQdA14n25YT/M6kqEREREZH6S0GpDhjSNhIPNwu7U7LZl5pjX1ky/W7nbEhPNK84EREREZF6SEGpDgj29eTClmEAzN2abF8Z0QZaDAbDBqv/Z2J1IiIiIiL1j4JSHTGyY5k24XCqVfjaT6Ewz4SqRERERETqJwWlOmJ4e3tQWn0gjdTsfPvKNmMgqAnkHoetM8wrTkRERESknlFQqiMah/jSqUkQhgELtqXYV7p7QI+b7curPjKvOBERERGRekZBqQ4ZWXzz2TnO0+963ARunnB4NRxZZ1JlIiIiIiL1i4JSHTKiuE34kl3HyC0osq8MiIQO4+3LCR+bVJmIiIiISP2ioFSHtIsOpGmoL/lFNpbsSj21oaSpw6bvIPeEOcWJiIiIiNQjCkp1iMVicYwqlep+F9sHojpDUR6s/8Kk6kRERERE6g8FpTqm5Dql+duOUmS12VdaLNC7+Aa0Cf8HNptJ1YmIiIiI1A8KSnVMr+ahhPh5kpZbyJoDaac2dL4avIMhbR/smW9egSIiIiIi9YCCUh3j4e7GRW0jgTLT77z84YLr7MtqFS4iIiIicl4UlOqgkR2Lr1PadhTDME5t6FU8/W7XHEjbX/uFiYiIiIjUEwpKddDA1hF4ebhx4HguO49mn9oQHg8thwIGrP6fafWJiIiIiNR1Ckp1kL+3BwPiwwGYuzW59MaSVuFrP4PCvFquTERERESkflBQqqPKbRMO0GY0BMfCyROw5QcTKhMRERERqfsUlOqoYe0jsVhgw6EMkjOcRo7c3KHnzfZlNXUQEREREakSBaU6KjLQh26xIYC9qUMp3W4Edy84shYOr6n94kRERERE6jgFpTpsRPHNZ0+bfhcQAR0usy+v+rh2ixIRERERqQcUlOqwkjbhK/akkpVXWHpjSVOHzd9DzvFarkxEREREpG5TUKrDWkUE0DLCn0KrwR87jpXe2LQXRHcBaz6s/9ycAkVERERE6igFpTrujN3vLJZTo0oJ/wc2ay1XJiIiIiJSdyko1XEji69TWrgjhYIiW+mNna4CnxBIPwC759V+cSIiIiIidZSCUh3XLTaE8ABvsvKKWLmvzLVIXn7Q7S/2ZbUKFxERERGpMAWlOs7NzcLw9pFAOdPvAHreYv+5ex6c2FuLlYmIiIiI1F0KSvVAyXVK87YexTCM0hvDWkGrYYBhv1ZJRERERETOSUGpHugfH46flztHMvLYciTz9B1Kmjqs+xwKcmu3OBERERGROkhBqR7w8XRnUOsIAOZsST59h9YjIbgZ5KXDlh9qtzgRERERkTpIQameKJl+N6e865Tc3KFX8bVKqz6CstPzRERERESkFAWleuKidpG4u1nYnpzFwRPlTK/rdiO4e0PSeji8ptbrExERERGpSxSU6olQfy96NQ8FzjCq5B8Gna6wL6tVuIiIiIjIWSko1SMjim8+O3drOdcpAfQqbuqw5QfISa2lqkRERERE6h5Tg9LixYsZN24cjRs3xmKxMGPGjFLbDcPg8ccfJyYmBl9fX4YPH86uXbvMKbYOGFl8nVLC/jTScgpO36FpD2jcDawFsPbTWq5ORERERKTuMDUo5eTk0LVrV/773/+Wu/2ll17irbfe4v3332flypX4+/szatQo8vLyarnSuiG2kR/togOx2gwWbE8pf6det9l/rp4KNmvtFSciIiIiUoeYGpTGjBnDs88+y+WXX37aNsMweOONN3j00UcZP348Xbp04dNPP+XIkSOnjTzJKSWjSnPLu04JoNOV4BsKGYmwa04tViYiIiIiUne47DVK+/btIzk5meHDhzvWBQcH06dPH1asWHHG1+Xn55OZmVnq0ZCUXKe0eNcx8grLGTHy9IVuf7Evq6mDiIiIiEi5XDYoJSfbGxJERUWVWh8VFeXYVp4XXniB4OBgxyM2NrZG63Q1nZoEERPsQ26BleV7ztCwoeetgAX2zIfje2q1PhERERGRusBlg1JVPfzww2RkZDgeBw8eNLukWmWxWE7dfHbLGabfNWoBrUfYlxP+r5YqExERERGpO1w2KEVH26eQHT1a+sv+0aNHHdvK4+3tTVBQUKlHQ1MSlOZtS8FmM8rfqaRV+PrPoaCcG9SKiIiIiDRgLhuUWrRoQXR0NPPnz3esy8zMZOXKlfTt29fEylxfnxZhBHp7kJqdz7qD6eXvFD8cQuIgLwM2fVur9YmIiIiIuDpTg1J2djbr169n/fr1gL2Bw/r160lMTMRisTBlyhSeffZZZs6cyaZNm7jxxhtp3Lgxl112mZlluzwvDzeGtosEYM6Zbj7r5ga9brUvJ3wExhlGnkREREREGiBTg9Lq1avp1q0b3bp1A+D++++nW7duPP744wA89NBD/P3vf+eOO+6gV69eZGdnM3v2bHx8fMwsu04Yca424QDdbgAPH0jeBIcSaqkyERERERHXZzGM+j2UkJmZSXBwMBkZGQ3qeqWsvEK6PzOXQqvB/AcG0yoioPwdZ9wF67+AztfAlWoXLiIiIiL1V2WygcteoyTnJ9DHk76twoFzjCr1us3+c+sMyD5W84WJiIiIiNQBCkr12Kk24We+7xRNukOTHmAtgLWf1FJlIiIiIiKuTUGpHhvR3h6U1h1M51hW/pl3LGkVvnoqWItqoTIREREREdemoFSPRQf70LVpMIYB87edZfpdx8vBtxFkHoKds2uvQBERERERF6WgVM85pt+d7TolTx/ofoN9OeHjWqhKRERERMS1KSjVcyM6RAOwdHcqOflnmVbX8xbAAnsXQuru2ilORERERMRFKSjVc22iAmjWyI+CIhtLdp2lq11oc2gzyr6sUSURERERaeAUlOo5i8XCSEf3u7NMv4NTTR3WfwkFOTVcmYiIiIiI61JQagBKrlNasCOFIqvtzDu2uggatYT8DNg4vZaqExERERFxPQpKDUCPuFBC/TxJzy0kYX/amXd0c4Oet9qXEz4Gw6idAkVEREREXIyCUgPg4e7GsOJ7Ks09W/c7gG7Xg4cvHN0MiX/WQnUiIiIiIq5HQamBONUmPBnjbCNFvqHQ+Sr7spo6iIiIiEgDpaDUQAxsHY63hxuH0k6yPTnr7Dv3us3+c+tPkJ1S88WJiIiIiLgYBaUGws/Lg4Gtw4EKTL9rfAE07QW2QljzSc0XJyIiIiLiYhSUGpCRxTefnbM1+dw7l7QKXzMVrGe5Ua2IiIiISD2koNSAXNQ+EosFNh/O5Ej6ybPv3PEy8AuHzMOwY1at1CciIiIi4ioUlBqQ8ABvejQLBWDetnNMv/Pwhu432pcTPqrhykREREREXIuCUgMzsmNx97st5whKAD1vAYsb7FsMx3bUcGUiIiIiIq5DQamBGVF8ndKfe4+TcbLw7DuHxEKbMfblhP+r4cpERERERFyHglID0yLcn/jIAIpsBn/sqEDr71632n9u+Arys2u2OBERERERF6Gg1ACNLL757DnbhAO0HAqNWkF+Jmz8poYrExERERFxDQpKDdCI4qD0x45j5BdZz76zm9upG9AmfAyGUcPViYiIiIiYT0GpAeraNITIQG+y84v4c++Jc7/gguvA0w9StsKB5TVfoIiIiIiIyRSUGiA3NwvD2pdMv6vAzWd9Q6Dz1fZltQoXERERkQZAQamBKmkTPnfrUWy2Ckyn6327/ee2nyGrAuFKRERERKQOU1BqoPq1CsPfy52jmflsOpxx7hdEd4bYC8FWBGs+qfkCRURERERMpKDUQHl7uDO4bQRQwe53cKqpw5qpYD3HPZhEREREROowBaUGbGTxzWcrHJQ6XAr+EZCVBNt/rcHKRERERETMpaDUgA1tG4m7m4UdR7M4cDzn3C/w8IbuN9mXEz6u2eJEREREREykoNSABft50qdFI6ASo0o9bwaLG+xfAinbarA6ERERERHzKCg1cCU3n51T0aAU3BTaXmxf1qiSiIiIiNRTCkoNXElQWr3/BCdyCir2opJW4Ru+hrzMGqpMRERERMQ8CkoNXNNQPzrEBGEzYP62Co4qtRgM4W2gIBs2flOzBYqIiIiImEBBSRyjShW+TsliOdUqPOFjMCpww1oRERERkTpEQUkY2dEelBbvOsbJAmvFXtT1WvD0h2PbYf/SGqxORERERKT2KSgJHWKCaBLiS16hjaW7Uyv2Ip9g6HKNfTnho5orTkRERETEBApKgsVicZp+l1zxF5Y0ddj2C2QeqYHKRERERETMoaAkAIwsDkrzt6VgtVXwmqOojtCsHxhWWDOt5ooTEREREallCkoCQK8WjQjy8eB4TgFrE9Mq/sLexU0d1kwDa2GN1CYiIiIiUtsUlAQAT3c3LmoXCVSi+x1Au3EQEAXZR2HbzzVUnYiIiIhI7VJQEocRHaIBmLMlGaOiLb89vKDHJPtywsc1U5iIiIiISC1TUBKHwW0j8HJ3Y//xXHanZFf8hT0mgcUdDiyDo1tqrD4RERERkdqioCQOAd4e9IsPA2BOZabfBTWGdmPtyxpVEhEREZF6QEFJSjnVJrwSQQlOtQrf8A3kZVRzVSIiIiIitUtBSUoZ0d4elNYfTCclM6/iL2w+ECLaQWEObPi6hqoTEREREakdCkpSSmSQDxfEhgAwd1slRpUsFuhV3Co84WOoaDMIEREREREXpKAkp6ny9LsuE8ArAFJ3wr7FNVCZiIiIiEjtUFCS04wsDkrLdx8nO7+o4i/0CYKu19qXEz6qgcpERERERGqHgpKcJj4ygBbh/hRYbSzacaxyLy6Zfrd9FmQcrv7iRERERERqgYKSnMZisThNv0uu3Isj20PcADCssGZqDVQnIiIiIlLzFJSkXCVBacH2FAqttsq9uHfxqNKaT6CooJorExERERGpeQpKUq7uzUIJ8/ciM6+IVftOVO7F7S6BwBjISYFtM2umQBERERGRGqSgJOVyd7MwrH0kUIXud+6e0GOSfTnh4+otTERERESkFigoyRmN6BAN2IOSUdn7IvWYBG4ekLgCkjdXf3EiIiIiIjVIQUnOaEB8OD6ebhxOP8nWpMzKvTgwGtqPsy+rVbiIiIiI1DEKSnJGvl7uDGodAcCcLZWcfgenWoVvnA4n06uvMBERERGRGqagJGd1qk14FYJSXH+IaA+FubDhq2quTERERESk5igoyVkNax+FmwW2JmVyKC23ci+2WE61Ck/4GGyVbDMuIiIiImISBSU5q0b+XvRs3gio4qhSlwngFQjHd8O+P6q3OBERERGRGqKgJOc08nym33kHwgUT7csJ/1eNVYmIiIiI1BwFJTmnkuuUVu47QUZuYeUPUNLUYccsSD9YjZWJiIiIiNQMBSU5p7gwf9pGBWK1GSzckVL5A0S0hRaDwLDBmqnVX6CIiIiISDVTUJIKKRlVmrM1uWoH6HW7/eeaT6Aov5qqEhERERGpGQpKUiElQWnRjmPkF1krf4C2F0NgY8hNha0/VXN1IiIiIiLVS0FJKqRzk2CigrzJKbCyfM/xyh/A3QN63mxfXvVR9RYnIiIiIlLNFJSkQtzcLKem322pQvc7gO43gZsnHFoFSRuqsToRERERkeqloCQVNqJDNADzth3FZjMqf4DAKOhwqX054eNqrExEREREpHopKEmFXdiyEQHeHhzLymfDofSqHaSkqcPGb+FkWrXVJiIiIiJSnRSUpMK8PdwZ0jYCgDlVufksQLMLIaoTFJ2E9V9WY3UiIiIiItVHQUkqpeQ6pblVDUoWy6kb0CZ8DDZbNVUmIiIiIlJ9FJSkUoa0jcTDzcLulGz2peZU7SCdrwbvIDixF/YuqN4CRURERESqgYKSVEqwrycXtgwDYG5Vbz7rHQAXXGdfXqWmDiIiIiLiehSUpNJGdjzPNuFwavrdztmQdqAaqhIRERERqT4KSlJpw9vbg9KaxDRSs/OrdpDw1tByCGDAmqnVVpuIiIiISHVQUJJKaxziS6cmQRgGLNiWUvUDlbQKX/spFOZVT3EiIiIiItVAQUmqZGTxzWfnVPU6JYA2oyGoKeQeh60zqqcwEREREZFqoKAkVVLSJnzJrlRyC4qqdhB3D+h5s3151UfVVJmIiIiIyPlTUJIqaRcdSNNQX/KLbCzZlVr1A3W/Edw84fBqOLKu+goUERERETkPLh+UsrKymDJlCnFxcfj6+tKvXz8SEhLMLqvBs1gsp6bfnU/3u4BI6HiZfXnFf8+/MBERERGRauDyQem2225j7ty5fPbZZ2zatImRI0cyfPhwDh8+bHZpDV7J9LsF249SZLVV/UB97rT/3PQtrJl2/oWJiIiIiJwnlw5KJ0+e5Pvvv+ell15i0KBBxMfH8+STTxIfH897771ndnkNXq/moYT4eZKWW8iaA2lVP1DTnjDk3/blXx+AvYuqp0ARERERkSpy6aBUVFSE1WrFx8en1HpfX1+WLl1a7mvy8/PJzMws9ZCa4eHuxkVtIwGYu/U8pt8BDH4IOl0FtiKYfgOk7q6GCkVEREREqsalg1JgYCB9+/blmWee4ciRI1itVj7//HNWrFhBUlJSua954YUXCA4OdjxiY2NrueqGZWRH+/S7OVuPYhhG1Q9kscD4/0LTXpCXAV9eA7knqqlKEREREZHKcemgBPDZZ59hGAZNmjTB29ubt956i4kTJ+LmVn7pDz/8MBkZGY7HwYMHa7nihmVg6wi8PNxIPJHLzqPZ53cwTx+49ksIjoUTe2D6jVBUUD2FioiIiIhUgssHpVatWrFo0SKys7M5ePAgq1atorCwkJYtW5a7v7e3N0FBQaUeUnP8vT0YEB8OwNzzuflsiYBIuO4b8AqA/Uvg1/vhfEaqRERERESqwOWDUgl/f39iYmJIS0vj999/Z/z48WaXJMVGdjg1/a5aRHWEq/4HFjdY9xmseKd6jisiIiIiUkEuH5R+//13Zs+ezb59+5g7dy5Dhw6lXbt23HzzzWaXJsWGtY/CYoGNhzJIzsirnoO2GQUjn7Mvz3kMdvxWPccVEREREakAlw9KGRkZTJ48mXbt2nHjjTcyYMAAfv/9dzw9Pc0uTYpFBHrTLTYEgLnbqmlUCeDCv0GPmwEDvrsVkjdV37FFRERERM7C5YPSNddcw549e8jPzycpKYl33nmH4OBgs8uSMkZ0iAaqoU24M4sFLn4ZWgyGwhz48lrIqsbji4iIiIicgcsHJakbStqEr9iTSmZeYfUd2N0TrvkEwuIh8xB8PREKT1bf8UVEREREyqGgJNWiVUQALSP8KbQaLNpxrHoP7hsK1023/zy8Bmb8DWy26n0PEREREREnCkpSbUYUd7+r1ul3JcJawYTPwc0TtvwIi/5T/e8hIiIiIlJMQUmqzcji65QWbk+hoKgGRnyaD4BLXrcvL3oRNn5b/e8hIiIiIoKCklSjbrEhhAd4k5VfxMp9x2vmTbrfAP3usS//NBkOrqqZ9xERERGRBk1BSaqNm5uF4e0jgRqafldi+JPQdixY8+Hr6yA9sebeS0REREQaJAUlqVbO1ykZhlEzb+LmDld8CFGdIecYfDkB8jJr5r1EREREpEFSUJJq1T8+HD8vd5Iy8th8uAbDi3cAXPc1BERBylb4/lawWWvu/URERESkQVFQkmrl4+nOoNYRAMzdmlyzbxbcFCZ+BR4+sGsOzHm0Zt9PRERERBoMBSWpdiXT7+bU5HVKJZr0gMvfty//+S6s/l/Nv6eIiIiI1HsKSlLtLmoXibubhe3JWRw8kVvzb9jxchhaPJr06z9gz8Kaf08RERERqdcUlKTahfp70at5KFBLo0oAg/4Bna8Bwwrf3gSpu2rnfUVERESkXlJQkhoxovjmszV+nVIJiwUufRti+0BeBnx5DeSeqJ33FhEREZF6R0FJasTI4uuUVu07QVpOQe28qacPTPgCQprBib3wzQ1QVEvvLSIiIiL1ioKS1IjYRn60iw7EZsCC7Sm198YBETDxG/AKhANL4df7oKbu5yQiIiIi9ZaCktSYkU43n61VUR3g6qlgcYN1n8Pyt2v3/UVERESkzlNQkhpTcp3S4l3HyCus5ZvBth4Bo16wL899HLb/WrvvLyIiIiJ1moKS1JhOTYKICfYht8DKst2ptV9An79Cz1sBA76/DZI21n4NIiIiIlInKShJjbFYLI6bz9b69Dt7ATDmRWg5BApz4atrIauWuvCJiIiISJ1WpaB08OBBDh065Hi+atUqpkyZwocfflhthUn9UBKU5m07itVmQlMFd0+4+hMIbwOZh+GriVBQCzfBFREREZE6rUpB6brrrmPhwoUAJCcnM2LECFatWsUjjzzC008/Xa0FSt3Wp0UYgT4epGYXsP5gmjlF+IbAdd+AbyM4shZm/A1sNnNqEREREZE6oUpBafPmzfTu3RuA6dOn06lTJ5YvX84XX3zBtGnTqrM+qeO8PNwY2jYSgDlmTL8r0aglTPgc3Dxh6wz44wXzahERERERl1eloFRYWIi3tzcA8+bN49JLLwWgXbt2JCUlVV91Ui+Yep2Ss+b9Ydwb9uXFL8HG6aaWIyIiIiKuq0pBqWPHjrz//vssWbKEuXPnMnr0aACOHDlCWFhYtRYodd+QthF4ulvYeyyH3SnZ5hbT7S/Qf4p9+afJkLjS1HJERERExDVVKSi9+OKLfPDBBwwZMoSJEyfStWtXAGbOnOmYkidSItDHk76twgEXGFUCGPYEtLsErAXw9XWQdsDsikRERETExVgMw6hSKzKr1UpmZiahoaGOdfv378fPz4/IyMhqK/B8ZWZmEhwcTEZGBkFBQWaX02B99ucBHpuxme7NQvjhrv5mlwMFOfC/0ZC8ESLaw61zwEfnh4iIiEh9VplsUKURpZMnT5Kfn+8ISQcOHOCNN95gx44dLhWSxHWMaG+/TmndwXRSsvJMrgbw8oeJX0NANBzbBt/dAtYis6sSERERERdRpaA0fvx4Pv30UwDS09Pp06cPr776KpdddhnvvfdetRYo9UN0sA9dmwZjGDB/W4rZ5dgFN4GJX4GHL+yeC3MeNbsiEREREXERVQpKa9euZeDAgQB89913REVFceDAAT799FPeeuutai1Q6g+X6X7nrEl3uPx9+/LK9yDhY3PrERERERGXUKWglJubS2BgIABz5szhiiuuwM3NjQsvvJADB3RhvJRvRIdoAJbuTiUn34WmuXW8DC56zL486yHYs8DUckRERETEfFUKSvHx8cyYMYODBw/y+++/M3LkSABSUlLUMEHOqE1UAHFhfhQU2Vi885jZ5ZQ28AHoci0YVpg+CY7tNLsiERERETFRlYLS448/zj/+8Q+aN29O79696du3L2AfXerWrVu1Fij1h8VicTR1cKnpdwAWC1z6FsReCPkZ8OU1kHPc7KpERERExCRVCkpXXXUViYmJrF69mt9//92xftiwYbz++uvVVpzUPyXXKS3YkUKR1WZyNWV4eMO1X0BIHKTtg+k3QFGB2VWJiIiIiAmqFJQAoqOj6datG0eOHOHQoUMA9O7dm3bt2lVbcVL/9IgLJdTPk/TcQhL2p5ldzun8w+G6b8A7CA4sg1/ug6rdakxERERE6rAqBSWbzcbTTz9NcHAwcXFxxMXFERISwjPPPIPN5mKjBOJSPNzdGFY8/W7O1mSTqzmDyPZw1VSwuMH6z2HZm2ZXJCIiIiK1rEpB6ZFHHuGdd97hP//5D+vWrWPdunU8//zzvP322zz22GPVXaPUM85twg1XHa1pPRxGv2hfnvckbPvF1HJEREREpHZZjCp8U23cuDHvv/8+l156aan1P/30E3fddReHDx+utgLPV2ZmJsHBwWRkZKgjn4vILSii29NzyS+y8du9A2kf48K/l18fsN9bydMPbpkNMV3NrkhEREREqqgy2aBKI0onTpwo91qkdu3aceLEiaocUhoQPy8PBraOAGDOFhfrflfW6Beh1UVQmAtfXguZSWZXJCIiIiK1oEpBqWvXrrzzzjunrX/nnXfo0qXLeRcl9d/Ikul321z0OqUS7h7265XC20LWEfjqWijINbsqEREREalhHlV50UsvvcTYsWOZN2+e4x5KK1as4ODBg8yaNataC5T66aL2kVgssPlwJkfST9I4xNfsks7MNwSu+xo+GgZJ62HGnXDVNHCrctNIEREREXFxVfqmN3jwYHbu3Mnll19Oeno66enpXHHFFWzZsoXPPvusumuUeig8wJsezUIBF7z5bHkatbTfY8nNE7b+BAufM7siEREREalBVWrmcCYbNmyge/fuWK3W6jrkeVMzB9f14eI9PD9rOwPiw/n8tj5ml1Mx67+EGX+zL1/+AXS91tx6RERERKTCaryZg0h1GNEhGoA/9x4n42ShydVU0AXXwYD77Msz/w6Jf5pbj4iIiIjUCAUlMU2LcH/iIwMoshn8sSPF7HIq7qLHod0lYC2Ar6+DE/vMrkhEREREqpmCkpiqpPvdnLpwnVIJNze44kP7PZVyj9s74eVlmF2ViIiIiFSjSnW9u+KKK866PT09/XxqkQZoRIco3v1jD4t2HCO/yIq3h7vZJVWMlz9M/Bo+ugiObYfvboGJ39jbiYuIiIhInVepEaXg4OCzPuLi4rjxxhtrqlaph7o2DSEy0Jvs/CL+3FvHblYc1BgmfgUevrB7Hvz+b7MrEhEREZFqUql//p46dWpN1SENlJubheEdovhyZSJztiQzuE2E2SVVTuNu9ml402+AVR9AeGvofbvZVYmIiIjIedI1SmK6UR3t3e9+WHuY/ak5JldTBR0uhWGP25d/+yfsnm9uPSIiIiJy3hSUxHQD48Pp2zKMk4VWHvxuAzZbtd3aq/YMuB+6TgTDCt9OgpTtZlckIiIiIudBQUlM5+Zm4aWruuDv5U7C/jSmLt9vdkmVZ7HAuDehWV/Iz4SvJkDOcbOrEhEREZEqUlASlxDbyI9/j20PwEuzt7PnWLbJFVWBhzdM+AJC4iBtP3zzFyjKN7sqEREREakCBSVxGdf1bsbA1uHkF9n4x7cbsNbFKXj+YXDddPAOgsTl8PMUMOrg5xARERFp4BSUxGVYLBZevLILgd4erEtM5+Mle80uqWoi28HVU8HiDhu+hKWvm12RiIiIiFSSgpK4lMYhvjx2SQcAXp27k11Hs0yuqIrih8OYF+3L85+CrTPNrUdEREREKkVBSVzO1T2bMqRtBAXFU/CKrDazS6qa3rdD7zvsyz/cAUfWmVuPiIiIiFSYgpK4HIvFwn+u6EKgjwcbDmXwweI6OgUPYNQL0GoYFJ2EryZC5hGzKxIRERGRClBQEpcUHezDk+M6AvDGvJ1sT840uaIqcvewX68U0Q6ykuCra6GgDt5UV0RERKSBUVASl3VF9yYMbx9FodXggekbKKyrU/B8gmHi1+AXBkkb4Me/gq2OfhYRERGRBkJBSVyWxWLh+Ss6EeLnyZYjmby7cI/ZJVVdoxb2eyy5e8G2n2HBM2ZXJCIiIiJnoaAkLi0y0IenLrVPwXt7wS62HMkwuaLzENcXLn3bvrz0NVj/lbn1iIiIiMgZKSiJy7u0a2NGd4ymyGafgldQVIenrXW9FgY+YF+e+Xc4sNzcekRERESkXApK4vIsFgvPXt6JRv5ebE/O4u0Fu8wu6fwMfRTaXwq2Qvj6ejixz+yKRERERKQMBSWpE8IDvHlmfCcA3v1jDxsPpZtb0Plwc4PLP4CYC+DkCfhyAuTV4SmFIiIiIvWQgpLUGWO7xHBJlxisxVPw8ousZpdUdV5+9k54gY0hdQd8OwmsRWZXJSIiIiLFFJSkTnl6fCfCA7zYlZLNG/Pq+BS8oBiY+BV4+sGeBTD7X2ZXJCIiIiLFFJSkTmnk78Wzl3UG4INFe1iXmGZyReep8QVwxYf25YSPYOWHppYjIiIiInYKSlLnjO4UzWUXNMZmwAPfbiCvsA5PwQNoPw6GP2lfnv1P2DXP1HJEREREREFJ6qgnL+1IZKA3e4/l8OqcHWaXc/76T4ELrgfDBt/dDCnbzK5IREREpEFTUJI6KcTPixeusE/B+3jpPlbvP2FyRefJYoFL3oBm/SA/094JLyfV7KpEREREGiwFJamzhrWP4qoeTTEM+Me3GzhZUMen4Hl4wYTPIbQ5pB+w32OpKN/sqkREREQaJAUlqdMeu6QD0UE+7D+ey0u/bze7nPPnHwbXTQfvYDj4J8y8BwzD7KpEREREGhwFJanTgn09+c+V9il4U5ft58+9x02uqBpEtIVrpoHFHTZ+DUteNbsiERERkQZHQUnqvCFtI7m2VywAD363gZz8enDj1lYXwcUv2ZcXPANvdYdf7oetP0FuHb8eS0RERKQOsBhG/Z7Xk5mZSXBwMBkZGQQFBZldjtSQrLxCRr+xhMPpJ7nhwjieuayT2SVVjwXP2keUDJvTSgvEdIWWg6HlEIi9ELz8zKpQREREpM6oTDZQUJJ6Y+muVP7yfysB+OK2PvSPDze5omqSlwH7l8HeP2DfIjhW5losdy+I7WMPTi2GQONu4O5hQqEiIiIirk1ByYmCUsPy6IxNfP5nIk1CfJk9ZSCBPp5ml1T9MpNg32J7aNr7B2QeLr3dOwiaD4AWxSNOEW3t7cdFREREGjgFJScKSg1LTn4Ro95YzKG0k0zs3cxxr6V6yzDg+B7Y90fxiNMSyEsvvU9ANLQYZA9NLQdDcNPar1NERETEBSgoOVFQanhW7DnOxI/+BOCTW3ozuE2EyRXVIpsVkjfaQ9PeRZC4AorySu/TqNWp0NR8IPg1MqNSERERkVpXb4KS1WrlySef5PPPPyc5OZnGjRszadIkHn30USwVnEqkoNQwPTlzC9OW7yc6yIff7xtEsG89nIJXEYV5cGiVPTTt/QOOrFVjCBEREWmw6k1Qev7553nttdf45JNP6NixI6tXr+bmm2/mueee45577qnQMRSUGqbcgiIufnMJ+4/ncnWPprx8dVezS3INagwhIiIiDVi9CUqXXHIJUVFR/N///Z9j3ZVXXomvry+ff/55ua/Jz88nPz/f8TwzM5PY2FgFpQYoYf8JrvlgBYYB/5vUk4vaRZldkusp1RhiEWQeKr1djSFERESkHqlMUHLpG87269eP+fPns3PnTgA2bNjA0qVLGTNmzBlf88ILLxAcHOx4xMbG1la54mJ6NW/Erf1bAPCv7zeRkVtockUuKCgGuk6Ay96F+zbD3Wtg7KvQfhz4hEB+JuyYBbP/Ce/2gVfbwQ93wLovIOPQOQ8vIiIiUle59IiSzWbj3//+Ny+99BLu7u5YrVaee+45Hn744TO+RiNK4iyv0MrFby5hb2oOl3drwusTLjC7pLqjIo0hwuKLR5vUGEJERERcX72Zevf111/z4IMP8vLLL9OxY0fWr1/PlClTeO2117jpppsqdAxdoyRrE9O46r3l2Az44IYejOoYbXZJdVOFG0MMsQenZn3B09ekYkVEREROV2+CUmxsLP/617+YPHmyY92zzz7L559/zvbt28/yylMUlATgP79t5/1FewgP8GLOfYNp5O9ldkl1nxpDiIiISB1TmWzg0t9acnNzcXMrfRmVu7s7NpvtDK8QKd+U4a2Zv+0ou1KyeWLmFt6e2M3skuo+n2Bod7H9AeU3hti/xP7g2VONIVoOsU/XU2MIERERcWEuHZTGjRvHc889R7NmzejYsSPr1q3jtdde45ZbbjG7NKljfDzdeeXqrlzx3nJ+3nCEMZ2iubhzjNll1S8ljSG6TgDDgON7YN8f9tC0bzHkpdsbQ+yYZd8/ILp4tKn4GqfgpmZWLyIiIlKKS0+9y8rK4rHHHuPHH38kJSWFxo0bM3HiRB5//HG8vCo2dUpT78TZK7/v4J2Fu2nk78Wc+wYRHuBtdkkNg6MxRPH1TWdtDDHEPvKkxhAiIiJSzerNNUrVQUFJnOUXWRn/zjK2J2cxplM0717fHYumf9U+58YQ+xbB4TVqDCEiIiI1TkHJiYKSlLX5cAaX/XcZRTaDtyZ249Kujc0uSSrTGKLlUIi5QI0hREREpNIUlJwoKEl53pi3kzfm7SLEz5M59w0iMtDH7JLEWXmNIZyVNIaIbA9+YU6PRqeWvQLULEJERERKUVByoqAk5Sm02hj/zjK2JmUyokMUH97QQ1PwXNWZGkOci7tX+QHqTMHKL0zT+0REROo5BSUnCkpyJtuSMrn0naUUWg1en9CVy7up61qdUNIYYt8SyDgEucedHifsP4tOVu3Ynn5nCVblrPNtBB66J5eIiEhdoaDkREFJzuadBbt4Zc5Ognw8mHPfYKKDNQWvXijIhZMnTg9Qpz2K1+ekgq2wau/lHVTxYOUXBr6h4OZevZ9XREREKkRByYmCkpxNkdXGFe8tZ+OhDIa2jeB/k3ppCl5DZBhQkF3xYFXyMKpy82sL+IZULFSVbPMOhjI3327wDANsRWUeVrAWln5uK7KH4JLnQY3tDxERaZAUlJwoKMm57Dqaxdi3llJgtfHSVV24pmes2SVJXWCzQX7GWUJVOcHqZFrV3sviXiZMneOaK++g4iBRTmiwFpYJEWWCxFn3sZYOJtayQaUKwaWq72dYq/67C2kGsRdCswvtrecj2imIiog0EApKThSUpCLeX7SH//y2nUBvD36/bxCNQ3RRv9QAa5G9EUVFQlXJuvxMs6uuWyzu4OZx6uHutGxxg8zDp48E+gTb28/H9rEHpybd1dhDRKSeUlByoqAkFWG1GVz1/nLWJaYzsHU4n97SW1PwxDUUFZS53uocwSr3OBTm2l9rcQM3T6fgUBwi3D1PLZ/x4V68X5nXOo7n9PqKHM/9bO9Vje95rv9u87PgUAIkroTEFXBoNRTmlN7HzRMaX2AfcSoZefIPr5Ffr4iI1C4FJScKSlJRe45lc/GbS8gvsvHCFZ2Z2LuZ2SWJVI21sHhkRdPJzslaBEc3QeKfpx7ZyafvFxbvFJz6Qlgr3adLRKQOUlByoqAklfHxkr08++s2/L3cmT1lELGN/MwuSURqk2FA+oHi0LTCPvJ0bNvp+/mFF1/jVByeYrqqVbyISB2goOREQUkqw2ozuPbDFSTsT6NfqzA+v7UPbm76V2ORBi33RPF0vRX2AHV4LVjzS+/j4QNNekKz4uucmvaydzeUuif3BBzfDam74MQe+/3S2l0MjVqaXZmIVAMFJScKSlJZ+1NzGP3mYvIKbTwzviM39G1udkki4kqK8uHIejjoNF3v5IkyO1kgssOp4NTsQgiO1XQ9V2EthLT9kLrTHoiO74LU3fafucfLf01kR2h/CbS7BKI763cpUkcpKDlRUJKqmLZsH0/+vBVfT3d+nzKIZmGagiciZ2AY9i/biSvgYHGTiBN7T98vsPGp6XrNLoSoTrr5cE0yDPvNpI/vOj0Mpe23t5k/k6Am9uvSwuLto0v7l5ZuSR8SZw9M7S+xd0vU71GkzlBQcqKgJFVhsxlM/OhPVu47Qe8Wjfj69gs1BU9EKi47xT7SVBKckjac/sXcK8A+Ra9ZX/vIU5Oe4B1gTr11WVG+PZg6h6HUnfblvIwzv87TH8LjIaw1hLe2h6KSn17+pffNPQE7f4ftv8Du+VB08tQ2/whoezG0HwctBoGHd818ThGpFgpKThSUpKoSj+cy+s3F5BZYefySDtwyoIXZJYlIXVWQC4fXFIenP+HgqtPvkWVxt0/pKpmq1+xCCIw2p15XYxiQfdQehlJ3nrqG6PguSE88/d5YDhYIiS0nDLWGoMZVmz5XkAN7FsC2X2Dnb6XDmFcgtBlpH21qPQK8A6v0cUWk5igoOVFQkvPx+Z8HeHTGZnw83Zh1z0BaRuhfe0WkGtiskLLtVIOIgysh4+Dp+4U2P3Uvp2YXQnjb+t32vSDX3kAhdVfpMJS6Gwqyzvw676DSISi8+NGoZc3ePNhaCPuX2EPT9l9Lt5Z394aWQ+zT89perHtxibgIBSUnCkpyPgzD4Ib/W8XS3an0iAtl+l/74q4peCJSEzIOlb6f09HNQJm/on1CiluSFzeJaNwNPH3MqLbqDAMyD5cThnaVHxZLWNzs1waFtzl9dCgg0vzmCjabfdRw+8/24HRiz6ltFjf776vkuqYQ3adPxCwKSk4UlOR8HUrLZfQbS8jOL+KRi9tz+yC1iBWRWpCXUdyWvPg6p8NroDC39D7uXvawVBKcYvuAf5g59ZaVn316EDq+C47vOf1zOPMJKT8MNWpRd67/MQw4tt0emLbNhOSNpbfHdIV24+yhKaKd+SFPpAFRUHKioCTV4etVifzrh014edin4MVHagqeiNQya6H9C7fzqFNOyun7hbc5dSPcZhfap5/V1Bdxm9U+ClTSTc55qlzWkTO/zs0DQlucHobCW4NfWP0LDmkH7FPztv9iD73O11Q1alXcdnwcNOlRv6dWirgABSUnCkpSHQzDYNLUBBbtPEbX2BC+v7MvHu76y0xETGQYkLavdHBK3XH6fv6Rp+7nFHshxHQBd8/KvVdeRjlhqHh0qOzNd535hTuFoTanAlFoXOVrqC9yUmHHLPto096FYC04tS0wpriD3iXQfGDD/TMSqUEKSk4UlKS6JGWcZOTri8nKK+Kh0W25a0i82SWJiJSWe+JUS/LElXBkbekv4gAevtC056kGEU17gU8wWIsg/UDxdLmdpa8hKm/kqoS7l33UynlUKKy1vfW2b2jNft66Lj8Lds21jzTtnFO6YYVPMLQZbW873moYeOl+fiLVQUHJiYKSVKfv1hziH99uwMvdjZ//PoC20Wr9KiIurDAPjqyztyQv6a53Mq3MThYIjoWsJLAVnvlYAdHlTJWLtzdY0A1Xz19RPuxbbL+mafssyE09tc3DF+KH2ZtBtBkFfo3Mq1OkjlNQcqKgJNXJMAxu+2Q187en0LlJMD/c1Q9PTcETkbrCZrOPFpUEp8QVkLb/1HYPH3sQKhuGwlqDj/4OrTU2qz3UbvvF3kUvPfHUNos7NB9gH2lqN9Z+PygRqTAFJScKSlLdjmbmMfL1xWScLOSBEW34+7DWZpckIlJ1Wcn2a41CYiGoqZoJuBrDgORN9ul5236BlC2ltzfpURyaxtlDrYiclYKSEwUlqQkz1h1myjfr8XS38NPkAXRorHNLRERqwfE9pzroHVxFqXttRbQ7da+mmAvqX/dAkWqgoOREQUlqgmEY3Pn5Gn7fcpT2MUH8NLk/Xh76V1gREalFWcmnQtO+xWArOrUtONY+Na/dJfaOh+4e5tUp4kIUlJwoKElNOZaVz8jXF5GWW8g9w1pz/4g2ZpckIiIN1cl02DUHtv0Mu+eVvqmvb6NTbcdbDgVPH9PKFDGbgpITBSWpSb9sPMLdX67D3c3CT5P706lJsNkliYhIQ1d4EvYstI807ZhVutOhpz+0HmG/rqn1CHsbcpEGREHJiYKS1LTJX6zl101JtI0KZObf++PtoTa5IiLiIqxFkLjcPtK0/VfIPHxqm5sntBxsn57XbiwERJpXp0gtUVByoqAkNe14dj4jX1/M8ZwCJg9txYOj2pldkoiIyOkMw34T4m2/2EebUnc6bbRAbB/79Lx2l0CjFqaVKVKTFJScKChJbZi9OYk7P1+LmwV+uKs/F8SGmF2SiIjI2R3bab9P07Zf7AHKWVSnUx30ojqpg57UGwpKThSUpLbc+/U6flp/hFYR/vx6z0B8PDUFT0RE6oiMQ7B9lj047V8GhvXUttDmxaFpHDTtrXttSZ2moOREQUlqS1pOASPfWMyxrHz+OqglD1/c3uySREREKi/3BOz4zT49b88CKMo7tc0/EtpdbA9OTXqAXyPz6hSpAgUlJwpKUpvmbj3K7Z+uxmKB7+7sS484/QUiIiJ1WEGOvd34tl9g5++Qn1F6e3AsRHcu/QiJ01Q9cVkKSk4UlKS23T99PT+sPUyLcH9m3TMQXy9NwRMRkXqgqAD2L7GPNO2eD+kHyt/POxiiO5UOTxHtwMO7dusVKYeCkhMFJaltGbmFjHxjEUcz87l1QAseu6SD2SWJiIhUv5PpcHQLJG8qfmyEY9vBWnD6vm4eEN729NEnTd2TWqag5ERBScywcEcKN09NwGKBb+7oS+8W+otAREQaAGuhve24c3hK2gh56eXvH9T09PAU2lxT96TGKCg5UVASs/zzu418s/ogzRr5MXvKQPy8PMwuSUREpPYZhv1Gt87hKXkTpO0vf3/vIHtL8rJT9zx9arVsqZ8UlJwoKIlZMvMKGf36Yo5k5HFT3zieGt/J7JJERERcR16G09S94vCUsu0sU/falBl96qKpe1JpCkpOFJTETEt2HeOG/1sFwJe396Ffq3CTKxIREXFh1kJI3VU6PCVvhJNp5e8f1KScrnvNda8nOSMFJScKSmK2f/+4iS9XJtI01JfZUwYR4K0peCIiIhVmGJB5pJype/vK398rsJyue+01dU8ABaVSFJTEbNn5RYx+YzGH0k5yXZ9mPH95Z7NLEhERqfvyMs8wdS//9H0t7uVP3fMPq/26xVQKSk4UlMQVLN+TynUfrQTg01t6M6hNhMkViYiI1EMlU/eObj4VnpI2wskT5e8f2LicrnstNHWvHlNQcqKgJK7iiZ8288mKAzQO9mH2fYMI8vE0uyQREZH6zzAgK6nMdU+b4MTe8vf3CnDqulf8M7IDePrWbt1SIxSUnCgoiavILShi9BtLSDyRy4Sesbx4VRezSxIREWm48rNOn7p3dOsZpu65nWHqnpo01TUKSk4UlMSVrNp3ggkfrsAwYOrNvRjaNtLskkRERKSEtQiO73JqHFEconKPl79/YMzp4Sm0Obi512rZUnEKSk4UlMTVPPPLVv5v6T6igryZM2UwwX6agiciIuKyDAOyksuZuren/P0t7uAfAQGREBAFgVH2nwFRxeuiT23zDqjdzyIKSs4UlMTVnCywMvatJexNzeGK7k147ZoLzC5JREREKis/yz5Vzzk8pWyForyKH8PT3ylIRZYJU07r/CPAXbcXqQ4KSk4UlMQVrTmQxtXvL8dmwEc39mREhyizSxIREZHzZbNCdgpkH3X6mVx6XVayfbkwtxIHttivhypvVCogEgKdgpV3EFgsNfYR67rKZANFUxET9IgL5faBLflg8V7+/eMmesaFEurvZXZZIiIicj7c3CEoxv44l/zs4vB0tHSwyiqzLicFDBvkHLM/jp7juB6+ZUanooqDVJkRK/8I8NB3j7PRiJKISfIK7VPw9hzLYfwFjXnz2m5mlyQiIiKuxma1N5MoCU9ZZYKVc6jKz6zcsX0blbmOqiRMlQlWvqH1ZpRKU++cKCiJK1t/MJ0r3l2GzYD3/9Kd0Z0q8C9QIiIiIuUpyD1DiDp6+pRAW1HFj+vuVSZIRZU/YuUfCZ4+Nff5qoGCkhMFJXF1L/++nf8u3EOYvxdz7htEWIC32SWJiIhIfWazwcm0M4epLKfrqvLSK3dsn+AyI1LFI1bBTaHTlTXycSpDQcmJgpK4uvwiK5e+vYwdR7MY2zmG/17f3eySREREROwK8+zXSTmPUp1p+p+14MzHCW0B966vtbLPRM0cROoQbw93Xr2mK+P/u4xfNyUxZuMRLunS2OyyREREROxT6UKa2R9nYxj20SdHZ78yIcovrFbKrU4KSiIuoFOTYCYPjeet+bt4bMZm+rQIIyJQU/BERESkjrBY7E0ffEMhoq3Z1VQLN7MLEBG7u4fG0yEmiLTcQh6dsYl6PitWRERExKUpKIm4CC8PN165uiue7hZ+33KUn9YfMbskERERkQZLQUnEhXRoHMQ9F7UG4ImZWziamWdyRSIiIiINk4KSiIu5c0grOjcJJuNkIf/+QVPwRERERMygoCTiYjzd7VPwvNzdmL89he/XHja7JBEREZEGR0FJxAW1jQ5kygj7FLynft7C6v0nTK5IREREpGFRUBJxUXcMbEm3ZiFk5RVx1fsr+Nf3G0nLOcuN3ERERESk2igoibgoD3c3pk7qxYSesQB8nXCQYa8t4tvVB3XdkoiIiEgNU1AScWEhfl68eFUXvruzL22jAjmRU8CD321kwod/sutoltnliYiIiNRbCkoidUDP5o345Z4BPDymHb6e7qzad4Ixby7hpdnbOVlgNbs8ERERkXpHQUmkjvB0d+Ovg1sx9/5BDG8fRZHN4N0/9jDi9UUs2H7U7PJERERE6hUFJZE6pmmoHx/f1JMPb+hB42AfDqWd5JZpq7nzszUkZZw0uzwRERGRekFBSaSOGtkxmrn3D+avg1ri7mZh9pZkhr+6iI+X7KXIajO7PBEREZE6TUFJpA7z9/bg4Yvb88vfB9AjLpScAivP/rqNce8sY21imtnliYiIiNRZCkoi9UD7mCC+/Wtf/nNFZ4J9PdmWlMmV7y3n3z9uIiO30OzyREREROocBSWResLNzcK1vZux4IHBXNWjKYYBX65M5KJX/+DHdYd07yURERGRSlBQEqlnwgK8eeXqrnx9x4XERwZwPKeA+77ZwHUfrWR3SrbZ5YmIiIjUCQpKIvXUhS3DmHXPQB4c1RZvDzdW7D3OmDcX8+qcHeQV6t5LIiIiImejoCRSj3l5uDF5aDzz7h/M0LYRFFoN3l6wm5GvL+aPHSlmlyciIiLishSURBqA2EZ+/G9SL97/S3eig3xIPJHLpKkJTP5iLUcz88wuT0RERMTlKCiJNBAWi4XRnWKY98Bgbh3QAnc3C79uSmLYq4uYumwfVpuaPYiIiIiUcPmg1Lx5cywWy2mPyZMnm12aSJ0U4O3BY5d0YObd/bkgNoTs/CKe+nkr4/+7lA0H080uT0RERMQlWAwX7xl87NgxrNZTF55v3ryZESNGsHDhQoYMGXLO12dmZhIcHExGRgZBQUE1WKlI3WOzGXyVkMiLv20nM68IiwX+0ieOf4xqS7Cvp9nliYiIiFSrymQDlw9KZU2ZMoVffvmFXbt2YbFYzrm/gpLIuR3Lyuf5Wdv4cd1hAMIDvHnskvZc2rVxhf47ExEREakLKpMNXH7qnbOCggI+//xzbrnlljN+ecvPzyczM7PUQ0TOLiLQm9cnXMCXt/WhZYQ/qdn53Pv1em74v1XsS80xuzwRERGRWlengtKMGTNIT09n0qRJZ9znhRdeIDg42PGIjY2tvQJF6rh+8eH8du9AHhjRBi8PN5buTmXU64t5fe5O3XtJREREGpQ6NfVu1KhReHl58fPPP59xn/z8fPLz8x3PMzMziY2N1dQ7kUran5rD4zO3sHjnMQCah/nxzGWdGNg6wuTKRERERKqmXl6jdODAAVq2bMkPP/zA+PHjK/w6XaMkUnWGYTBrUzJP/byFlCz7P0Bc2rUxj17SnshAH5OrExEREamcenmN0tSpU4mMjGTs2LFmlyLSYFgsFsZ2iWH+A4OZ1K85bhaYueEIw15ZxKcr9uveSyIiIlJv1YmgZLPZmDp1KjfddBMeHh5mlyPS4AT6ePLkpR35afIAujQNJiu/iMd/2sLl7y5j8+EMs8sTERERqXZ1IijNmzePxMREbrnlFrNLEWnQOjcN5se7+vPM+I4Eenuw8VAGl76zlCdnbiErr9Ds8kRERESqTZ25RqmqdI2SSM1Iyczj2V+3MXPDEQAiA715fFwHxnaO0b2XRERExCXVy2uURMS1RAb58NbEbnx2a2+ah/mRkpXP3V+u46apCRw4rnsviYiISN2moCQi52Vg6whmTxnElOGt8XJ3Y/HOY4x8fTFvz99FfpHuvSQiIiJ1k4KSiJw3H093pgxvw+wpAxkQH05+kY1X5+5kzJtLWL4n1ezyRERERCpNQUlEqk3LiAA+u7U3b157AeEB3uw9lsN1H63kvm/Wcywr/9wHEBEREXERCkoiUq0sFgvjL2jC/AcGc8OFcVgs8OO6wwx79Q++WHkAm+69JCIiInWAut6JSI3acDCdf/+4iS1HMgHo1iyEZy/rRMfGwSZXJiIiIg2Nut6JiMvoGhvCT5P788S4DgR4e7AuMZ1xby/lmV+2kp1fZHZ5IiIiIuVSUBKRGufh7sbN/Vsw7/7BjO0cg82A/1u6j+GvLmL25iTq+cC2iIiI1EEKSiJSa6KDffjv9d2ZdnMvmjXyIzkzjzs/X8utn6zm4Ilcs8sTERERcdA1SiJiirxCK/9duJv3F+2h0Grg4+nG3y9qze0DW+LloX/DkephGAbrD6Yza1MSC7anEBnow18Ht2RwmwgsFovZ5YmISC2rTDZQUBIRU+1OyebRGZv4c+8JAOIjA3jusk70aRlmcmVSVxmGwbqD6czamMRvm5M5nH7ytH06Nwlm8tB4RnaIws1NgUlEpKFQUHKioCTi+gzDYMb6wzz7yzaO5xQAcFWPpjw8ph1hAd4mVyd1gXM4mrUpiSMZeY5tfl7uDG8fxaiO0aw/mMbnfyZystAKQJuoACYPjWds5xg83DWSKSJS3ykoOVFQEqk7MnILefH37Xy5MhGAYF9PHh7Tjmt6xupf/eU0JeHo141J/FYmHPl7uTOsfRQXd45hSNsIfDzdHdtO5BQwddk+pi3bT1Zx58W4MD/uGtKKy7s11dRPEZF6TEHJiYKSSN2zNjGNR37czLYk+72XesSF8tzlnWgXrf+GGzqbrXjkaFP54Wh4B3s4GtymdDgqT2ZeIZ+tOMDHS/aSllsIQONgH/46uBUTesWe8/UiIlL3KCg5UVASqZuKrDamLd/Pa3N3kltgxd3Nwm0DWnDv8Nb4eXmYXZ7UIudwNGtTEknnEY7Kk1tQxJcrE/lw8V5SsvIBCA/w5vaBLbj+wjgCvHW+iYjUFwpKThSUROq2I+knefrnrczekgxAkxBfnhjXgZEdo02uTGpSSTj6dWMSv20uPxyN7RzDoCqGo/LkFVr5bs0h3vtjj6MBRLCvJ7f0b8Gkfs0J9vOslvcRERHzKCg5UVASqR8WbD/K4z9t4VCa/Qvs8PZRTBnemvYxQbjr+qV6wR6O0vh1Y/Jp4SjA24Ph7SO5uJrDUXkKrTZmrDvMe3/sYW9qjuP9b+gbx60DWhCuBiMiInWWgpITBSWR+uNkgZW3F+ziw8V7KbLZ/9fl7+XOBc1C6N4slO5xoXSPDdW//NchrhKOymO1GczalMR/F+5me3IWAD6ebkzs3Yw7BrUkJti3VusREZHzp6DkREFJpP7ZeTSLl2bv4M+9x8ku7lrmLD4ygO7NQugRF0r3ZqG0ighQ1zwXYrMZrE1M49dNSfy2KZnkzNPD0dgujRnYOtwlGirYbAbzt6fwzsLdbDiYDoCXuxtX9mjK3wa3olmYn7kFiohIhSkoOVFQEqm/rDaDnUezWJuYxpoDaaxLTGdf8VQpZ0E+HnRrFuoITl1jgwn00ahTbTpbOAr09nA0ZHCVcFQewzBYujuVdxbsZuU++w2S3d0sjO/amLuGtiI+MtDkCkVE5FwUlJwoKIk0LMez81mXmM6axDTWHkhjw6F08gptpfZxs0CbqEBHcOoRF0pcmB8Wi0adqlNJOPplYxKzN5cfjsZ2jmFgm3C8PVwzHJ1Jwv4TvLNgN4t2HgPAYoExnaKZPDSejo2DTa5ORETOREHJiYKSSMNWaLWxPSmLNQdOsDYxnTUH0hwdzZw18veie7MQ+3VOzULp2jQEX6+69eXdFdhsBmsS0/j1DOFoRMnIUR0MR+XZeCiddxbsZs7Wo451F7WLZPLQeHrEhZpYmYiIlEdByYmCkoiUdTQzj7UH0libmMbaxHQ2HcqgwFp61MnDzUL7mCB6xIXSrfh6pyYhvhp1KodzOPptcxJHM/Md2+pjOCrPjuQs/rtwN79sPEJxnxH6tQrj7ovi6dsyTOeNiIiLUFByoqAkIueSX2Rly5FMR3hacyCt1Jf9EpGB3o7pet3jQunUJKjefvE/l4qEo7FdYhjQuv6Go/LsS83hvT9288Paw47OjD3iQrl7aDxD2kYoMImImExByYmCkohUlmEYHMnIY82BNEd42nok0/HFt4SXuxudmgQ5glOPuFCignxMqrrm2WwGqw+kMWvTGcJRR/s1Rw0tHJXnUFouHy7ey9cJBykoso9WdmwcxN1D4xnVMVpdGEVETKKg5ERBSUSqw8kCKxsPpbM2Md0+Ze9AGsdzCk7br0mIb/F1Tvbpeu1jgvB0dzOh4urhHI5mbUoiJcspHPkUjxwpHJ1RSmYeHy/dx+d/HiC3wArY29dPHtqKcV0a41GHzw0RkbpIQcmJgpKI1ATDMEg8kWsfdUpMY82BdHYkZ1Jm0AkfTze6NA1xdNfr3iyEsABvc4quoJJw9OvGI/y2Ofm0cDSyQzRju0TTP17hqKLScgqYumwfU5fvJyvPfu+vZo38+NuQVlzRvYn+HEVEaomCkhMFJRGpLdn5RWw4mM7aA2msSbTf1ynjZOFp+zUP86N7s1C6xYXSo1kobaMDcTd5KpbVZrB6/4niaXUKRzUlM6+Qz1Yc4P+W7uNE8YhkTLAPdwxqybW9mqnToohIDVNQcqKgJCJmsdkM9qZms/ZAumPkaVdK9mn7+Xu5c0GzEMe1Tt1jQwn2q/kb4paEo1+Lw9GxcsLRJV1i6B8fjpeHpohVp9yCIr5adZAPF+9xXOsVHuDFrQNa8pcLm+mGyCIiNURByYmCkoi4kozcQtYdtLclX3sgjfUH08nOLzptv/jIAMd1Tt2bhdIqIqBaGgBYbQYJTiNHzuEoyMeDkR2jGdtZ4ai25BdZ+W7NId77Yw+H0uz39wr29WRSv+bc3L85IX5eJlcoIlK/KCg5UVASEVdmtRnsPJrlaEu+LjGdfak5p+0X5ONBN8d1TqF0jQ2u8KiDwpHrK7TamLn+CP/9Yzd7j9l///5e7vylbxy3DWhJRKBrX9cmIlJXKCg5UVASkbrmeHY+6xLTWVPcXW/DoXTyCkvfENfNAm2iAu1tyYun7DUP83Pcp6ckHP26MYnZW84QjrrE0L+VwpErsdoMZm9O5p2Fu9mWlAmAt4cbE3s3445BLWkc4mtyhSIidZuCkhMFJRGp6wqtNrYnZbHmwAlHe/KSaVrOGvl70b1ZCOEB3szblkJq9qlwFOzrycgOUVyscFQnGIbBgu0pvL1gN+sPpgPg6W7hyu5N+duQVsSF+ZtboIhIHaWg5ERBSUTqo5TMPMd0vbWJ6Ww6lEGBtfSoU0k4Gtslhn4KR3WSYRgs33Octxfs4s+9JwD7aOKlXRszeWg8raMCTa5QRKRuUVByoqAkIg1BfpGVLUcyWXsgjeSMPAa0Dlc4qmdW7z/BOwt388eOY451oztGc/dF8XRqEmxiZSIidYeCkhMFJRERqU82Hcrgvwt3M3tLsmPdkLYR/P2ieHrENTKxMhER16eg5ERBSURE6qOdR7N4d+FuZm44gq34b/K+LcO4+6J4+rUKczT2EBGRUxSUnCgoiYhIfbY/NYf3F+3h+7WHKLTa/0rv1iyEu4fGc1G7SAUmEREnCkpOFJRERKQhOJJ+kg8X7+WrVYnkF9kbe7SPCeLuofGM7hSNezXcsFhEpK5TUHKioCQiIg3Jsax8Pl66l89XHCCnwApAywh/Jg+J59ILGuPprgYfItJwKSg5UVASEZGGKD23gKnL9jN12T4y84oAiG3ky52DW3FVj6Z4e7ibXKGISO1TUHKioCQiIg1ZVl4hn/+ZyMdL9nI8pwCAqCBv7hjUiut6N8PXS4FJRBoOBSUnCkoiIiJwssDK1wmJfLBoL8mZeQCE+Xtxy4AW3Ng3jkAfT5MrlNpmtRlk5RWScdL+yDxZZP9Zal3xzzz7tqzi5wVFNrrGhtA/PpwB8eF0aByk6+CkTlBQcqKgJCIickp+kZXv1xzmvUW7OXjiJAA+nm5EBHrj7+WBv7f9EeDt7vTcvXidB35exdsc+3ng5+VOQPFzXQNVu/IKrWUCTXHIyT0VbkpvKyKzeF1WflG11RHs60m/VmH0jw+nf3w4zcP81HFRXJKCkhMFJRERkdMVWW38vPEI7yzYzZ5jOdV2XC8PN/y9TgUr/zJBKqA4eNkD16lQ5udVentJSPPyqN/By2YzyMovOhVmygSeM4/yFJGZZx/ZOV9+Xu4E+XgS7Gt/BPl6EOTrWWZd8U8fD4L9PDEMWLXvBEt3p/LnnuOnha4mIb70j7cHp36twokI9D7vOkWqg4KSEwUlERGRM7PZDPamZpNxsojcgiJy8ovIzreSk19ETvHznHwr2fn27Y5t+UXF6+zbquMLe3m83N3KBKszj26VDWgl+/o7hbKaCF75RVZHoCkJNZllp63llglAefZRn6z8Is73m5ibhXKCjUdxsHEKOc5hp3g50MfzvP9Miqw2Nh7OYPnuVJbuTmXtgXQKrKXPh3bRgcWjTWH0bhFGgLfHeb2nSFUpKDlRUBIREal5hVZbcbiyOkJUTv6poJVT4LzuVBBzDl72YFazwcvT3VIqPPl5F492lTPN0N/LnSKbcdp1OiXPSwJPXuH51+rt4VYqzDgHGuegUxKGSoJQsK8n/l4euLnQ9UG5BUUk7E9zBKctRzJLbfdws9CtWQj9WoUzoHU4F8SGaMqm1BoFJScKSiIiInVPodVGbr6V7AKn0aviEOUcqkqFsrLrCk69Jr+GgpezQJ9TozjljewE+3mWGeU5NcXNx7P+dh88kVPAij3HWbo7lWW7U0k8kVtqu7+XO31ahtGvVRgDWofTNipQ1zdJjVFQcqKgJCIiIs7BK9cRrk5NKTzTlEN3N0vpsHPaNDb7zwAfD3V9q6CDJ3JZVjzatHzPcU4Ut60vER7gbQ9N8eH0bx1OkxBfkyqV+khByYmCkoiIiIhrstkMtidnOYLTqn0nOFloLbVP8zA/Rxvyvq3CCPHzMqlaqQ8UlJwoKImIiIjUDQVFNtYlprFsdyrL9hxn/cF0rLZTX1UtFujUONjRGKJX80b1etqiVD8FJScKSiIiIiJ1U1ZeISv3niieppfKzqPZpbZ7ebjRMy7Ucf+mzk2CNQVSzkpByYmCkoiIiEj9kJKZx7I9qSzbfZxlu1NJysgrtT3Ix4O+Tje+bRnur8YQUoqCkhMFJREREZH6xzAM9qbmONqQL99znKy80je+jQn2KW5DHkb/VuFEBvmYVG39V2S1kZpdQFLGSY5m5pGckUdSZh5HM/JIzswjMtCHtyZ2M7tMBSVnCkoiIiIi9Z/VZrDpcIb9+qbdqazen3bajW/bRAXYg1N8OH1aNiLQx9OkauuWkwVWkjPznEJQPskZJ0nOzCM50758LCsf21lSRdNQX5b+86LaK/oMFJScKCiJiIiINDx5hVZW709z3L9p85EMnL/1urtZ6No02N6GPD6cbs1C8fJoWDe+NQyDtNxCkjPySM48aQ9AmXnFISifoxn2cJRZZqTuTNzdLEQGehMV5ENMsA9RQT5EB9uXG4f40qt5oxr+ROemoOREQUlERERE0nNL3/h2//HSN7719XSnd4tGjuDULjoQtzrcGKLQaiMlq3jkp7wAlHmSo5n5FFTwZsx+Xu5EFwef6CAfooKdwlBxMAoL8Hb5ZhoKSk4UlERERESkrENpuSzffdzRUS81u/SNbxv5e5268W18OLGN/Eyq9HTZ+UX2UaDi63+OFk+LS87IL17O43hOPhX9lh/m71U6ABX/dASjYB8CvT3qRWMMBSUnCkoiIiIicjaGYbDjaBZLd9lHm1buO0FuQekb3zZr5Oe4f1O/VuE08q/+G9/abAbHcwocYSe5uBlCUoY9DCUXN0nIzq/YVDhPd4tjxKck+DhPiYsO8iEyyBtvj4ZzLyoFJScKSiIiIiJSGQVFNjYcSmfpLvto07rEdIrKdCro2DiIAfHh9IsPp3fzRvh6nT1s5BdZScnML26KcKobXLLTz5SsPAqtFftqHujt4RjtKXVNkNMoUCM/rzo9fbAmKCg5UVASERERkfORnV/Eqn3HWbrrOMv3pLI9OavUdi93N7rHhdC/VThRQT7F3eDySk2PO5FTcIajl2axQESA91kDUHSQD/7eHjXxUes9BSUnCkoiIiIiUp1SsvLsjSGKp+odKXPj2zPx9nBzBKDTpsEVB6CIQG883RtW973aVJlsoCgqIiIiIlIJkYE+jL+gCeMvaIJhGOw/nsvS3ams2JNKTr613AAUHeRDiJ9nvWiI0FAoKImIiIiIVJHFYqFFuD8twv254cI4s8uRaqRxPRERERERkTIUlERERERERMpQUBIRERERESlDQUlERERERKQMBSUREREREZEyFJRERERERETKUFASEREREREpQ0FJRERERESkDAUlERERERGRMhSUREREREREylBQEhERERERKUNBSUREREREpAwFJRERERERkTIUlERERERERMpw+aB0+PBh/vKXvxAWFoavry+dO3dm9erVZpclIiIiIiL1mIfZBZxNWloa/fv3Z+jQofz2229ERESwa9cuQkNDzS5NRERERETqMZcOSi+++CKxsbFMnTrVsa5FixYmViQiIiIiIg2BS0+9mzlzJj179uTqq68mMjKSbt268dFHH531Nfn5+WRmZpZ6iIiIiIiIVIZLjyjt3buX9957j/vvv59///vfJCQkcM899+Dl5cVNN91U7mteeOEFnnrqqdPWKzCJiIiIiDRsJZnAMIxz7msxKrKXSby8vOjZsyfLly93rLvnnntISEhgxYoV5b4mPz+f/Px8x/PDhw/ToUOHGq9VRERERETqhoMHD9K0adOz7uPSI0oxMTGnhZz27dvz/fffn/E13t7eeHt7O54HBARw8OBBAgMDsVgsNVZrRWRmZhIbG8vBgwcJCgoytRapG3TOSGXpnJHK0jkjlaVzRqrCVc4bwzDIysqicePG59zXpYNS//792bFjR6l1O3fuJC4ursLHcHNzO2darG1BQUH6H4tUis4ZqSydM1JZOmeksnTOSFW4wnkTHBxcof1cupnDfffdx59//snzzz/P7t27+fLLL/nwww+ZPHmy2aWJiIiIiEg95tJBqVevXvz444989dVXdOrUiWeeeYY33niD66+/3uzSRERERESkHnPpqXcAl1xyCZdcconZZVQLb29vnnjiiVLXUImcjc4ZqSydM1JZOmeksnTOSFXUxfPGpbveiYiIiIiImMGlp96JiIiIiIiYQUFJRERERESkDAUlERERERGRMhSUREREREREylBQqkX//e9/ad68OT4+PvTp04dVq1aZXZKY4IUXXqBXr14EBgYSGRnJZZdddtqNlfPy8pg8eTJhYWEEBARw5ZVXcvTo0VL7JCYmMnbsWPz8/IiMjOTBBx+kqKioNj+KmOQ///kPFouFKVOmONbpnJGyDh8+zF/+8hfCwsLw9fWlc+fOrF692rHdMAwef/xxYmJi8PX1Zfjw4ezatavUMU6cOMH1119PUFAQISEh3HrrrWRnZ9f2R5FaYLVaeeyxx2jRogW+vr60atWKZ555BueeXzpnZPHixYwbN47GjRtjsViYMWNGqe3VdY5s3LiRgQMH4uPjQ2xsLC+99FJNf7TyGVIrvv76a8PLy8v43//+Z2zZssW4/fbbjZCQEOPo0aNmlya1bNSoUcbUqVONzZs3G+vXrzcuvvhio1mzZkZ2drZjnzvvvNOIjY015s+fb6xevdq48MILjX79+jm2FxUVGZ06dTKGDx9urFu3zpg1a5YRHh5uPPzww2Z8JKlFq1atMpo3b2506dLFuPfeex3rdc6IsxMnThhxcXHGpEmTjJUrVxp79+41fv/9d2P37t2Off7zn/8YwcHBxowZM4wNGzYYl156qdGiRQvj5MmTjn1Gjx5tdO3a1fjzzz+NJUuWGPHx8cbEiRPN+EhSw5577jkjLCzM+OWXX4x9+/YZ3377rREQEGC8+eabjn10zsisWbOMRx55xPjhhx8MwPjxxx9Lba+OcyQjI8OIiooyrr/+emPz5s3GV199Zfj6+hoffPBBbX1MBwWlWtK7d29j8uTJjudWq9Vo3Lix8cILL5hYlbiClJQUAzAWLVpkGIZhpKenG56ensa3337r2Gfbtm0GYKxYscIwDPv/qNzc3Izk5GTHPu+9954RFBRk5Ofn1+4HkFqTlZVltG7d2pg7d64xePBgR1DSOSNl/fOf/zQGDBhwxu02m82Ijo42Xn75Zce69PR0w9vb2/jqq68MwzCMrVu3GoCRkJDg2Oe3334zLBaLcfjw4ZorXkwxduxY45Zbbim17oorrjCuv/56wzB0zsjpygal6jpH3n33XSM0NLTU303//Oc/jbZt29bwJzqdpt7VgoKCAtasWcPw4cMd69zc3Bg+fDgrVqwwsTJxBRkZGQA0atQIgDVr1lBYWFjqfGnXrh3NmjVznC8rVqygc+fOREVFOfYZNWoUmZmZbNmypRarl9o0efJkxo4dW+rcAJ0zcrqZM2fSs2dPrr76aiIjI+nWrRsfffSRY/u+fftITk4udc4EBwfTp0+fUudMSEgIPXv2dOwzfPhw3NzcWLlyZe19GKkV/fr1Y/78+ezcuROADRs2sHTpUsaMGQPonJFzq65zZMWKFQwaNAgvLy/HPqNGjWLHjh2kpaXV0qex86jVd2ugUlNTsVqtpb6gAERFRbF9+3aTqhJXYLPZmDJlCv3796dTp04AJCcn4+XlRUhISKl9o6KiSE5OduxT3vlUsk3qn6+//pq1a9eSkJBw2jadM1LW3r17ee+997j//vv597//TUJCAvfccw9eXl7cdNNNjt95eeeE8zkTGRlZaruHhweNGjXSOVMP/etf/yIzM5N27drh7u6O1Wrlueee4/rrrwfQOSPnVF3nSHJyMi1atDjtGCXbQkNDa6T+8igoiZho8uTJbN68maVLl5pdiriwgwcPcu+99zJ37lx8fHzMLkfqAJvNRs+ePXn++ecB6NatG5s3b+b999/npptuMrk6cUXTp0/niy++4Msvv6Rjx46sX7+eKVOm0LhxY50z0mBp6l0tCA8Px93d/bQOVEePHiU6OtqkqsRsd999N7/88gsLFy6kadOmjvXR0dEUFBSQnp5ean/n8yU6Orrc86lkm9Qva9asISUlhe7du+Ph4YGHhweLFi3irbfewsPDg6ioKJ0zUkpMTAwdOnQota59+/YkJiYCp37nZ/t7KTo6mpSUlFLbi4qKOHHihM6ZeujBBx/kX//6F9deey2dO3fmhhtu4L777uOFF14AdM7IuVXXOeJKf18pKNUCLy8vevTowfz58x3rbDYb8+fPp2/fviZWJmYwDIO7776bH3/8kQULFpw2vNyjRw88PT1LnS87duwgMTHRcb707duXTZs2lfqfzdy5cwkKCjrty5HUfcOGDWPTpk2sX7/e8ejZsyfXX3+9Y1nnjDjr37//abcd2LlzJ3FxcQC0aNGC6OjoUudMZmYmK1euLHXOpKens2bNGsc+CxYswGaz0adPn1r4FFKbcnNzcXMr/bXQ3d0dm80G6JyRc6uuc6Rv374sXryYwsJCxz5z586lbdu2tTrtDlB78Nry9ddfG97e3sa0adOMrVu3GnfccYcREhJSqgOVNAx/+9vfjODgYOOPP/4wkpKSHI/c3FzHPnfeeafRrFkzY8GCBcbq1auNvn37Gn379nVsL2n1PHLkSGP9+vXG7NmzjYiICLV6bkCcu94Zhs4ZKW3VqlWGh4eH8dxzzxm7du0yvvjiC8PPz8/4/PPPHfv85z//MUJCQoyffvrJ2LhxozF+/Phy2/h269bNWLlypbF06VKjdevWavVcT910001GkyZNHO3Bf/jhByM8PNx46KGHHPvonJGsrCxj3bp1xrp16wzAeO2114x169YZBw4cMAyjes6R9PR0IyoqyrjhhhuMzZs3G19//bXh5+en9uD13dtvv200a9bM8PLyMnr37m38+eefZpckJgDKfUydOtWxz8mTJ4277rrLCA0NNfz8/IzLL7/cSEpKKnWc/fv3G2PGjDF8fX2N8PBw44EHHjAKCwtr+dOIWcoGJZ0zUtbPP/9sdOrUyfD29jbatWtnfPjhh6W222w247HHHjOioqIMb29vY9iwYcaOHTtK7XP8+HFj4sSJRkBAgBEUFGTcfPPNRlZWVm1+DKklmZmZxr333ms0a9bM8PHxMVq2bGk88sgjpVo065yRhQsXlvsd5qabbjIMo/rOkQ0bNhgDBgwwvL29jSZNmhj/+c9/ausjlmIxDKdbLouIiIiIiIiuURIRERERESlLQUlERERERKQMBSUREREREZEyFJRERERERETKUFASEREREREpQ0FJRERERESkDAUlERERERGRMhSUREREREREylBQEhERERERKUNBSUREXN6xY8f429/+RrNmzfD29iY6OppRo0axbNkyACwWCzNmzDC3SBERqVc8zC5ARETkXK688koKCgr45JNPaNmyJUePHmX+/PkcP37c7NJERKSe0oiSiIi4tPT0dJYsWcKLL77I0KFDiYuLo3fv3jz88MNceumlNG/eHIDLL78ci8XieA7w008/0b17d3x8fGjZsiVPPfUURUVFju0Wi4X33nuPMWPG4OvrS8uWLfnuu+8c2wsKCrj77ruJiYnBx8eHuLg4Xnjhhdr66CIiYiIFJRERcWkBAQEEBAQwY8YM8vPzT9uekJAAwNSpU0lKSnI8X7JkCTfeeCP33nsvW7du5YMPPmDatGk899xzpV7/2GOPceWVV7Jhwwauv/56rr32WrZt2wbAW2+9xcyZM5k+fTo7duzgiy++KBXERESk/rIYhmGYXYSIiMjZfP/999x+++2cPHmS7t27M3jwYK699lq6dOkC2EeGfvzxRy677DLHa4YPH86wYcN4+OGHHes+//xzHnroIY4cOeJ43Z133sl7773n2OfCCy+ke/fuvPvuu9xzzz1s2bKFefPmYbFYaufDioiIS9CIkoiIuLwrr7ySI0eOMHPmTEaPHs0ff/xB9+7dmTZt2hlfs2HDBp5++mnHiFRAQAC33347SUlJ5ObmOvbr27dvqdf17dvXMaI0adIk1q9fT9u2bbnnnnuYM2dOjXw+ERFxPQpKIiJSJ/j4+DBixAgee+wxli9fzqRJk3jiiSfOuH92djZPPfUU69evdzw2bdrErl278PHxqdB7du/enX379vHMM89w8v/buX+X1MI4juNvToiLRm1JKA4N5iAURYPuLoHLsXbJpbGxIShaEqo/QBrFRNrCzV2iRWiJCPoHbJMGCe7WxdO92/3hvb1fcIaHw/nC8yyHD8/zfN/e2NnZIQzDXzUlSdIMMyhJkv5J+Xye8XgMQCwW4/39fer9+vo6j4+PrKysfHqC4PvvbzAYTH03GAxYXV39GM/Pz7O7u0uz2aTT6XBzc8Pr6+tvnJkkaRbYHlySNNNGoxHVapVarUahUCCZTHJ/f0+j0aBSqQCQzWbp9/sUi0Xi8TiLi4scHR2xvb1NJpMhDEOCIGA4HPLw8MDp6elH/W63y8bGBqVSiVarxd3dHVdXVwBcXFyQSqVYW1sjCAK63S5LS0ssLCz8jaWQJP1BBiVJ0kxLJBJsbW1xeXnJ8/Mzk8mEdDpNvV7n8PAQgPPzcw4ODmg2mywvL/Py8kK5XOb29paTkxPOzs6IxWLkcjn29vam6h8fH3N9fc3+/j6pVIp2u00+nwcgmUzSaDR4enpibm6Ozc1Ner3e1I6UJOn/ZNc7SdKX9aNueZIkgXeUJEmSJOkTg5IkSZIkRXhHSZL0ZXn6XJL0M+4oSZIkSVKEQUmSJEmSIgxKkiRJkhRhUJIkSZKkCIOSJEmSJEUYlCRJkiQpwqAkSZIkSREGJUmSJEmK+AbyxXVHrtc2WAAAAABJRU5ErkJggg==\n" + }, + "metadata": {} + } + ] }, { "cell_type": "code", "source": [ "# testing\n", - "\n", "target_text = \"Would you like to tell me your name because \"\n", "context = torch.tensor([tokenizer.encode(target_text)], dtype=torch.long, device=device)\n", - "generated_output = tokenizer.decode(m.generate(context, max_new_tokens=10)[0].tolist())\n", - "print(generated_output)" + "generated_output = tokenizer.decode(m.generate(context, max_new_tokens=10))\n", + "print(target_text, generated_output)" ], "metadata": { - "id": "TTIOcKHshxsH" + "id": "TTIOcKHshxsH", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "1be93194-d8fe-4a1e-ac84-06384242c10f" }, - "execution_count": null, + "execution_count": 23, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Would you like to tell me your name because the full comic throw in so said disinfect mess V\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "torch.cuda.empty_cache()" + ], + "metadata": { + "id": "v8y1w-wVYCts" + }, + "execution_count": 30, "outputs": [] } ]