DEEP_Character_LSTM_from_Scratch_JAX.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "OD2YpM0yclp0"
   },
   "outputs": [],
   "source": [
    "import jax.numpy as jnp\n",
    "from jax import grad, jit\n",
    "from jax import random\n",
    "\n",
    "from textwrap import wrap\n",
    "\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Pf1oQlfWdPEV",
    "outputId": "03e6ceaa-3bae-4116-c607-859bf924a09c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26 unique characters\n",
      "208000 total characters\n"
     ]
    }
   ],
   "source": [
    "alphabet_data = 'abcdefghijklmnopqrstuvwxyz'*8000\n",
    "\n",
    "chars = set(list(alphabet_data))\n",
    "\n",
    "vocab_size = len(chars)\n",
    "\n",
    "char_to_ix = {ch:i for i,ch in enumerate(chars)}\n",
    "ix_to_char = {i:ch for i,ch in enumerate(chars)}\n",
    "\n",
    "print(f'{vocab_size} unique characters')\n",
    "print(f'{len(alphabet_data)} total characters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lVYosrYdc6GU",
    "outputId": "7fbad808-093d-4311-c44a-6114dac0b02d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSTM with 11 hidden units\n",
      "Single layer LSTM has 1984 total parameters\n",
      "Will train on character sequences of length 8\n",
      "\n",
      "Training data sample: \n",
      "\n",
      "               abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqr               \n",
      "               stuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghij               \n",
      "                    klmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqr                    \n",
      "\n"
     ]
    }
   ],
   "source": [
    "hidden_size = 11\n",
    "\n",
    "seq_length = 8\n",
    "\n",
    "total_params = 5 * hidden_size * vocab_size + 4 * hidden_size**2 + 4 * hidden_size + vocab_size \n",
    "\n",
    "print(f'LSTM with {hidden_size} hidden units')\n",
    "print(f'Single layer LSTM has {total_params} total parameters')\n",
    "print(f'Will train on character sequences of length {seq_length}')\n",
    "print()\n",
    "txt = wrap(alphabet_data[len(alphabet_data)//2:(len(alphabet_data)//2)+200])\n",
    "txt = [line.center(100) for line in txt]\n",
    "txt = '\\n'.join(txt)\n",
    "print('Training data sample: \\n')\n",
    "print(txt)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "wVrhDpOidnkE"
   },
   "outputs": [],
   "source": [
    "def initialize_lstm_weights(key,n_h,n_x):\n",
    "    \"\"\"\n",
    "    Initializes lstm layer weights with normal distribution\n",
    "\n",
    "    Args:\n",
    "        key (PRNG Key): Key controlling jax.random random number generation\n",
    "        n_h (int): Dimension of the LSTM hidden layer\n",
    "        n_x (int): Dimension of the input embedding vectors\n",
    "\n",
    "    Returns:\n",
    "        params (dict): Collection of all LSTM layer parameters\n",
    "\n",
    "        Wxc,Wxu,Wxf,Wxo (jax array): matrices shape=(n_h,n_x) act on batchs\n",
    "                                     of input column vectors (batch_size,n_x)\n",
    "\n",
    "        Whc,Whu,Whf,Who (jax array): matrices shape=(n_h,n_h) act on hidden states\n",
    "\n",
    "        bc,bu,bf,bo (jax array): bias vectors shape=(n_h,1)\n",
    "\n",
    "        grad_mems(dict): Collection of memory arrays for gradients , needed for ADAM algorithm\n",
    "        sqrd_mems(dict): Collection of memory arrays for squared gradients\n",
    "    \"\"\"\n",
    "\n",
    "    subkeys=random.split(key,9) # need to call random.noraml with new key each time\n",
    "\n",
    "    params, grad_mems, sqrd_mems = dict(),dict(),dict()\n",
    "\n",
    "    params['Wxc'] = random.normal(subkeys[1],(n_h,n_x))*0.01 # input to cell state\n",
    "    params['Wxu'] = random.normal(subkeys[2],(n_h,n_x))*0.01 # input to update\n",
    "    params['Wxf'] = random.normal(subkeys[3],(n_h,n_x))*0.01 # input to forget\n",
    "    params['Wxo'] = random.normal(subkeys[4],(n_h,n_x))*0.01 # input to output\n",
    "\n",
    "    params['bc'] = jnp.zeros((n_h, 1)) # hidden bias\n",
    "    params['bu'] = jnp.zeros((n_h, 1)) # forget bias\n",
    "    params['bf'] = jnp.zeros((n_h, 1)) # update bias\n",
    "    params['bo'] = jnp.zeros((n_h, 1)) # output bias\n",
    "\n",
    "    params['Whc'] = random.normal(subkeys[5],(n_h,n_h))*0.01 # hidden to cell\n",
    "    params['Whu'] = random.normal(subkeys[6],(n_h,n_h))*0.01 # hidden to update\n",
    "    params['Whf'] = random.normal(subkeys[7],(n_h,n_h))*0.01 # hidden to forget\n",
    "    params['Who'] = random.normal(subkeys[8],(n_h,n_h))*0.01 # hidden to output\n",
    "\n",
    "    for parameter in params.keys():\n",
    "\n",
    "        shape = params[parameter].shape\n",
    "\n",
    "        grad_mems[parameter]=jnp.zeros(shape)\n",
    "        sqrd_mems[parameter]=jnp.zeros(shape)\n",
    "\n",
    "    return params, grad_mems, sqrd_mems\n",
    "\n",
    "def initialize_dense_weights(key,n_h,n_x):\n",
    "    \"\"\"\n",
    "    Initializes final dense layer weights for character probabilities\n",
    "\n",
    "    Args:\n",
    "        key (PRNG Key): Key controlling jax.random random number generation\n",
    "        n_h (int): Dimension of the LSTM hidden layer\n",
    "        n_x (int): Vocab size for softmatx outputs\n",
    "\n",
    "    Returns:\n",
    "        params (dict): Collection of all LSTM layer parameters\n",
    "\n",
    "        Why (jax array): matrix shape=(n_x,n_h) maps hidden states to output\n",
    "                         probabilities over characters in the vocab\n",
    "\n",
    "        by (jax array): bia vectors shape=(n_x,1)\n",
    "\n",
    "        grad_mems(dict): Collection of memory arrays for gradients , needed for ADAM algorithm\n",
    "        sqrd_mems(dict): Collection of memory arrays for squared gradients\n",
    "    \"\"\"\n",
    "\n",
    "    key,subkey=random.split(key)\n",
    "\n",
    "    params,grad_mems,sqrd_mems = dict(),dict(),dict()\n",
    "\n",
    "    params['Why'] = random.normal(subkey,(n_x,n_h))*0.01 # hidden to output\n",
    "    params['by'] = jnp.zeros((n_x, 1)) # output bias\n",
    "\n",
    "    for parameter in params.keys():\n",
    "\n",
    "        shape = params[parameter].shape\n",
    "\n",
    "        grad_mems[parameter]=jnp.zeros(shape)\n",
    "        sqrd_mems[parameter]=jnp.zeros(shape)\n",
    "\n",
    "    return params, grad_mems, sqrd_mems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NtiIlmbjfRYj",
    "outputId": "536bbdd3-c3db-4385-f6da-2367dfd86795"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:jax._src.lib.xla_bridge:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer 0:  dict_keys(['Wxc', 'Wxu', 'Wxf', 'Wxo', 'bc', 'bu', 'bf', 'bo', 'Whc', 'Whu', 'Whf', 'Who'])\n",
      "Layer 1:  dict_keys(['Wxc', 'Wxu', 'Wxf', 'Wxo', 'bc', 'bu', 'bf', 'bo', 'Whc', 'Whu', 'Whf', 'Who'])\n",
      "Layer 2:  dict_keys(['Why', 'by'])\n",
      "0 Wxc (11, 26) 286\n",
      "0 Wxu (11, 26) 286\n",
      "0 Wxf (11, 26) 286\n",
      "0 Wxo (11, 26) 286\n",
      "0 bc (11, 1) 11\n",
      "0 bu (11, 1) 11\n",
      "0 bf (11, 1) 11\n",
      "0 bo (11, 1) 11\n",
      "0 Whc (11, 11) 121\n",
      "0 Whu (11, 11) 121\n",
      "0 Whf (11, 11) 121\n",
      "0 Who (11, 11) 121\n",
      "1 Wxc (11, 11) 121\n",
      "1 Wxu (11, 11) 121\n",
      "1 Wxf (11, 11) 121\n",
      "1 Wxo (11, 11) 121\n",
      "1 bc (11, 1) 11\n",
      "1 bu (11, 1) 11\n",
      "1 bf (11, 1) 11\n",
      "1 bo (11, 1) 11\n",
      "1 Whc (11, 11) 121\n",
      "1 Whu (11, 11) 121\n",
      "1 Whf (11, 11) 121\n",
      "1 Who (11, 11) 121\n",
      "2 Why (26, 11) 286\n",
      "2 by (26, 1) 26\n",
      "2996\n"
     ]
    }
   ],
   "source": [
    "key = random.PRNGKey(0)\n",
    "\n",
    "all_params, all_grads, all_sqrd = {},{},{}\n",
    "\n",
    "key,subkey = random.split(key)\n",
    "\n",
    "all_params[0],all_grads[0],all_sqrd[0] = initialize_lstm_weights(subkey,hidden_size,vocab_size)\n",
    "\n",
    "key,subkey = random.split(key)\n",
    "\n",
    "all_params[1],all_grads[1],all_sqrd[1] = initialize_lstm_weights(subkey,hidden_size,hidden_size)\n",
    "\n",
    "key,subkey = random.split(key)\n",
    "\n",
    "all_params[2],all_grads[2],all_sqrd[2] = initialize_dense_weights(subkey,hidden_size,vocab_size)\n",
    "\n",
    "print('Layer 0: ',all_params[0].keys())\n",
    "print('Layer 1: ',all_params[1].keys())\n",
    "print('Layer 2: ',all_params[2].keys())\n",
    "\n",
    "total_params = 0\n",
    "\n",
    "for k1,v1 in all_params.items():\n",
    "\n",
    "    for k2,v2 in v1.items():\n",
    "\n",
    "        print(k1,k2,v2.shape,v2.size)\n",
    "\n",
    "        total_params += v2.size\n",
    "\n",
    "print(total_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "nfih6PcffY4S"
   },
   "outputs": [],
   "source": [
    "def get_mini_batch(mini_batch_size,seq_length,char_to_ix,data):\n",
    "    '''\n",
    "    Generator that continuously yields mini-batches of text sequences\n",
    "\n",
    "    Args:\n",
    "        mini_batch_size (int): Number of sequences each mini batch\n",
    "        seq_length (int): Number of characters per sequence\n",
    "        char_to_ix (dict): Dict mapping characters to their indices\n",
    "\n",
    "        data (str): String containing all the characters in the data set\n",
    "\n",
    "    Returns:\n",
    "        inputs (jax array): batch of sequence indices shape = (mini_batch_size,seq_length)\n",
    "        targets (jax array): shifted sequences with next character targets\n",
    "\n",
    "        epoch (int): counter tracking number of times have looped over entire data set\n",
    "    '''\n",
    "\n",
    "    p = 0\n",
    "    epoch = 0\n",
    "\n",
    "    batch_character_size = mini_batch_size*(seq_length)\n",
    "\n",
    "    while True:\n",
    "\n",
    "        if p+batch_character_size+1>=len(data):\n",
    "            p=0\n",
    "            epoch = epoch + 1\n",
    "\n",
    "            # need to reset hprev,cprev if it loops?\n",
    "\n",
    "        inputs,targets = [],[]\n",
    "\n",
    "        for _ in range(mini_batch_size):\n",
    "\n",
    "            inputs.append([char_to_ix[ch] for ch in data[p:p+seq_length]])\n",
    "            targets.append([char_to_ix[ch] for ch in data[p+1:p+seq_length+1]])\n",
    "\n",
    "            p += seq_length\n",
    "\n",
    "        inputs = jnp.array(inputs)\n",
    "        targets = jnp.array(targets)\n",
    "\n",
    "        yield inputs,targets,epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 191
    },
    "id": "CDlL1PFOio0A",
    "outputId": "7125b704-c5ee-40a8-e61a-cccbb739217b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DeviceArray([[ 8,  7, 18,  6, 23, 15, 25, 13],\n",
       "             [ 2,  0, 17,  9,  3, 21, 12, 20],\n",
       "             [ 5, 22, 11, 19, 16,  1,  4, 14],\n",
       "             [24, 10,  8,  7, 18,  6, 23, 15]], dtype=int32)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DeviceArray([[ 7, 18,  6, 23, 15, 25, 13,  2],\n",
       "             [ 0, 17,  9,  3, 21, 12, 20,  5],\n",
       "             [22, 11, 19, 16,  1,  4, 14, 24],\n",
       "             [10,  8,  7, 18,  6, 23, 15, 25]], dtype=int32)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "m = 4 seq_length = 8\n"
     ]
    }
   ],
   "source": [
    "batch_gen = get_mini_batch(4,seq_length,char_to_ix,alphabet_data)\n",
    "\n",
    "inp,targ,epoch_test = next(batch_gen)\n",
    "\n",
    "display(inp,targ,epoch_test)\n",
    "\n",
    "print(f'm = {inp.shape[0]} seq_length = {inp.shape[1]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "kkNHK-6EjYHh"
   },
   "outputs": [],
   "source": [
    "def encode_inputs(inputs,vocab_size):\n",
    "    '''\n",
    "    One-hot encodes batches of characater sequences\n",
    "\n",
    "    Args:\n",
    "        inputs (jax array): batch of characters shape=(mini_batch_size,seq_length)\n",
    "        vocab_size (int): number of characters in the vocab\n",
    "    Returns:\n",
    "        xs (dict): dictionary of jax arrays, indexed by time t in [0,1,...,seq_length]\n",
    "        xs[t]: jax array at time t, shape = (vocab_size,mini_batch_size)\n",
    "    '''\n",
    "\n",
    "    xs = {}\n",
    "\n",
    "    xs[-1] = None  # need hs and xs dicts to have same length = seq_length + 1\n",
    "\n",
    "    mini_batch_size = inputs.shape[0]\n",
    "    seq_length = inputs.shape[1]\n",
    "\n",
    "    for t in range(seq_length):\n",
    "        xs[t] = jnp.zeros((vocab_size,mini_batch_size))\n",
    "        xs[t] = xs[t].at[inputs[:,t],jnp.arange(mini_batch_size)].set(1) # batch of one-hot vectors for time t\n",
    "\n",
    "    return xs\n",
    "\n",
    "def sigmoid(z):\n",
    "    # sigmoid activation for LSTM gates\n",
    "    return 1.0/(1.0 + jnp.exp(-z))\n",
    "\n",
    "def softmax(y):\n",
    "    #computes softmax probabilities over characters\n",
    "    return jnp.exp(y) / jnp.sum(jnp.exp(y),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "kkNHK-6EjYHh"
   },
   "outputs": [],
   "source": [
    "def lstm_forward(inputs,hprev,cprev,params):\n",
    "    '''\n",
    "    Computes forward pass for single LSTM layer\n",
    "\n",
    "    Args:\n",
    "        inputs (dict): dict of jax arrays each of shape (input_size,mini_batch_size)\n",
    "        hprev (jax array): column vector (n_h,1) for starting hidden state\n",
    "        cprev (jax array): column vector (n_h,1) for starting cell state\n",
    "        params (dict): dictionary of matrices W and biases b for this layer\n",
    "    Returns:\n",
    "        states (tuple): Cache of variables needed layer for backprop\n",
    "                        xs (dict): input vectors each time step\n",
    "                        hs (dict): hidden states\n",
    "                        cs (dict): cell states\n",
    "                        c_tildes (dict): candidate cell states\n",
    "        gates (tuple): Cache of the LSTM gates\n",
    "                        gamma_us (dict): update gates\n",
    "                        gamma_fs (dict): forget gates\n",
    "                        gamma_os (dict): output gates\n",
    "    '''\n",
    "\n",
    "    # dims\n",
    "    mini_batch_size = inputs[0].shape[1]\n",
    "    seq_length = len(inputs) - 1\n",
    "    #print(\"SEQ LENGTH \",seq_length)\n",
    "\n",
    "    vocab_size = params['Wxc'].shape[1]\n",
    "\n",
    "    # unpack params\n",
    "    Wxc = params['Wxc']\n",
    "    Wxu = params['Wxu']\n",
    "    Wxf = params['Wxf']\n",
    "    Wxo = params['Wxo']\n",
    "\n",
    "    Whc = params['Whc']\n",
    "    Whu = params['Whu']\n",
    "    Whf = params['Whf']\n",
    "    Who = params['Who']\n",
    "\n",
    "    bc = params['bc']\n",
    "    bu = params['bu']\n",
    "    bf = params['bf']\n",
    "    bo = params['bo']\n",
    "\n",
    "    xs = inputs  # inputs should be given as dict with seq_length entries, each item = (n_x,m) array of vectors\n",
    "\n",
    "    hs, cs, c_tildes = {},{},{}\n",
    "\n",
    "    gamma_us, gamma_fs, gamma_os = {}, {}, {}\n",
    "\n",
    "    hs[-1] = jnp.tile(hprev,(1,mini_batch_size))\n",
    "    cs[-1] = jnp.tile(cprev,(1,mini_batch_size))\n",
    "\n",
    "    for t in range(seq_length):\n",
    "\n",
    "        zc = jnp.dot(Wxc,xs[t]) + jnp.dot(Whc,hs[t-1]) + bc  # linear activation for candidate cell state C~\n",
    "        zu = jnp.dot(Wxu,xs[t]) + jnp.dot(Whu,hs[t-1]) + bu  # linear activation for update gate\n",
    "        zf = jnp.dot(Wxf,xs[t]) + jnp.dot(Whf,hs[t-1]) + bf  # linear activation for forget gate\n",
    "        zo = jnp.dot(Wxo,xs[t]) + jnp.dot(Who,hs[t-1]) + bo  # linear activation for output gate\n",
    "\n",
    "        c_tildes[t] = jnp.tanh(zc) # canidate for new c state\n",
    "\n",
    "        gamma_us[t] = sigmoid(zu)\n",
    "        gamma_fs[t] = sigmoid(zf)\n",
    "        gamma_os[t] = sigmoid(zo)\n",
    "\n",
    "        cs[t] = jnp.tanh(jnp.multiply(c_tildes[t],gamma_us[t]) + jnp.multiply(cs[t-1],gamma_fs[t]))  # tanh here is import!!!\n",
    "\n",
    "        hs[t] = jnp.multiply(cs[t],gamma_os[t]) # hidden state\n",
    "\n",
    "    gates = (gamma_us,gamma_fs,gamma_os)\n",
    "\n",
    "    states = (xs,hs,cs,c_tildes)\n",
    "\n",
    "    return states, gates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "kkNHK-6EjYHh"
   },
   "outputs": [],
   "source": [
    "def loss_function(targets,hidden_states,params):\n",
    "    '''\n",
    "    Computes categorical cross entropy loss over predicted characters\n",
    "\n",
    "    Args:\n",
    "        targets (jax array): batch of target sequences shape=(mini_batch_size,seq_length)\n",
    "        hidden_states (dict): collection of hidden states from last LSTM layer\n",
    "        params (dict): jax arrays with layer weights\n",
    "    Returns:\n",
    "        loss (float): categorical cross entropy loss averaged over batch sequences\n",
    "        p_cache (tuple): cache with logits and softmax probabilies\n",
    "                        ys (dict): set of unnormalized log probs on characters\n",
    "                        ps (dict): set of softmax probabilities\n",
    "    '''\n",
    "\n",
    "    mini_batch_size=targets.shape[0]\n",
    "    seq_length = targets.shape[1]\n",
    "\n",
    "    Why = params['Why']\n",
    "    by = params['by']\n",
    "\n",
    "    ys, ps = {},{}\n",
    "\n",
    "    loss = 0\n",
    "\n",
    "    for t in range(seq_length):\n",
    "\n",
    "        ys[t] = jnp.dot(Why, hidden_states[t]) + by # unnormalized log probabilities for next chars\n",
    "\n",
    "        ps[t] = softmax(ys[t]) # probabilities for next chars  #  ps[t] should be shape (vocab_size,mini_batch_size)\n",
    "\n",
    "        loss += jnp.mean(jnp.log(jnp.sum(jnp.exp(ys[t]),axis=0)) - ys[t][targets[:,t],jnp.arange(mini_batch_size)])\n",
    "        #loss += -jnp.mean(jnp.log(ps[t][targets[:,t],jnp.arange(mini_batch_size)]))\n",
    "\n",
    "    p_cache = (ys,ps)\n",
    "\n",
    "    return loss, p_cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UxIUNno4oiKU",
    "outputId": "7099575e-2bf2-43c4-cfe4-85a9e35f0cb0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 (26, 4)\n",
      "<class 'tuple'> 4 <class 'tuple'> 3\n",
      "<class 'tuple'> 4 <class 'tuple'> 3\n",
      "<class 'dict'> 9\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#initial zeros for hidden and cell states\n",
    "h = jnp.zeros((hidden_size,1))\n",
    "c = jnp.zeros((hidden_size,1))\n",
    "\n",
    "#one-hot encode batch of input tokens\n",
    "inputs_onehot = encode_inputs(inp,26)\n",
    "\n",
    "print(len(inputs_onehot)-1,inputs_onehot[0].shape)\n",
    "\n",
    "#dicts to hold lstm states and lstm gates each layer \n",
    "s_cache,g_cache = {},{}\n",
    "\n",
    "#forward pass through first lstm layer \n",
    "s_cache[0], g_cache[0] = lstm_forward(inputs_onehot,h,c,all_params[0])\n",
    "\n",
    "print(type(s_cache[0]),len(s_cache[0]),type(g_cache[0]),len(g_cache[0]))\n",
    "\n",
    "hs_layer0 = s_cache[0][1]  # hidden states from layer zero\n",
    "\n",
    "#forward pass through second lstm layer \n",
    "s_cache[1], g_cache[1] = lstm_forward(hs_layer0,h,c,all_params[1])\n",
    "\n",
    "print(type(s_cache[0]),len(s_cache[0]),type(g_cache[0]),len(g_cache[0]))\n",
    "\n",
    "hs_layer1 = s_cache[1][1]  # hidden states from layer 1\n",
    "\n",
    "print(type(hs_layer1),len(hs_layer1))\n",
    "\n",
    "# pass final hidden states to dense + softmax to get probs and loss \n",
    "test_loss,p_cache = loss_function(targ,hs_layer1,all_params[2])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2WrNAE9ao4fO",
    "outputId": "3e1fcb4b-322e-4e6c-a756-a313604cac1a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DeviceArray(26.064775, dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "H6uPaSKyrW5A",
    "outputId": "477f659c-ad2f-47d7-dbfa-00ae1d3c3661"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'tuple'> 2\n",
      "<class 'tuple'> 3\n",
      "<class 'tuple'> 4\n"
     ]
    }
   ],
   "source": [
    "print(type(p_cache),len(p_cache))\n",
    "print(type(g_cache[0]),len(g_cache[0]))\n",
    "print(type(s_cache[0]),len(s_cache[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "M9URF-wmqYhY"
   },
   "outputs": [],
   "source": [
    "def loss_backward(targets,probs_cache,final_hidden_states,dense_params):\n",
    "    '''\n",
    "    Backpropagation through final loss layer\n",
    "\n",
    "    Args:\n",
    "        targets (jax_array): batch of target indices shape=(mini_batch_size,seq_length)\n",
    "        probs_cache (tuple): (ys,hs) logits and softmax probabilities\n",
    "        final_hidden_states (dict): hidden states output by last LSTM layer\n",
    "        dense_params (dict): parameters for the loss layer\n",
    "    Returns:\n",
    "        dhs (dict): cache of hidden state grads to pass backwards\n",
    "        grads (dict): dictionary with parameter gradients for this layer\n",
    "    '''\n",
    "\n",
    "    #unpack caches\n",
    "    ys,ps = probs_cache\n",
    "    hs = final_hidden_states  # dictionary with hidden states from last layer\n",
    "\n",
    "    #weights for dense output to softmax layer\n",
    "    Why = dense_params['Why']\n",
    "    by = dense_params['by']\n",
    "\n",
    "    # dims\n",
    "    mini_batch_size = targets.shape[0]  # targets = (m,seq_length) rows of tokens each sample\n",
    "    seq_length = targets.shape[1]\n",
    "\n",
    "    vocab_size = Why.shape[0]\n",
    "\n",
    "    #initialize grads\n",
    "    dWhy,dby = jnp.zeros_like(Why), jnp.zeros_like(by)\n",
    "\n",
    "    #cache to pass gradients back to lstm_backwards\n",
    "    dhs = {}\n",
    "\n",
    "    #backward pass\n",
    "    for t in reversed(range(seq_length)):\n",
    "        dy = jnp.copy(ps[t])\n",
    "\n",
    "        dy = dy.at[targets[:,t],jnp.arange(mini_batch_size)].add(-1) #backprop into y\n",
    "\n",
    "        dWhy += jnp.dot(dy, hs[t].T)\n",
    "        dby += jnp.sum(dy,axis=1,keepdims=True)\n",
    "\n",
    "        dhs[t] = jnp.dot(Why.T, dy)\n",
    "\n",
    "    grads = dict()\n",
    "\n",
    "    grads['Why']=dWhy\n",
    "    grads['by']=dby\n",
    "\n",
    "    for parameter in grads.keys():\n",
    "        grads[parameter] = jnp.clip(grads[parameter], -5, 5) # clip to mitigate exploding gradients\n",
    "\n",
    "    return dhs, grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "M9URF-wmqYhY"
   },
   "outputs": [],
   "source": [
    "def lstm_layer_backward(dh_next_layer,gates_cache,states_cache,params):\n",
    "    '''\n",
    "    Computes backprop for a single LSTM layer\n",
    "\n",
    "    Args:\n",
    "        dh_next_layer (dict): cache of intermediate derivatives from higher layer\n",
    "        gates_cache (dict): gates cache from forward pass of this layer\n",
    "        states_cache (dict): cache of foward pass variables this layer\n",
    "        params (dict): set of weights W biases b for this layer\n",
    "    Returns:\n",
    "        dh_previous_layer (dict): cache of hidden state derivatives to pass backwards\n",
    "        grads (dict): dictionary of gradients corresponding to params\n",
    "    '''\n",
    "    #unpack caches\n",
    "\n",
    "    gamma_us,gamma_fs,gamma_os = gates_cache\n",
    "\n",
    "    xs,hs,cs,c_tildes = states_cache\n",
    "\n",
    "    # dh_next_layer # dictionary of derivatives from backpass of higher layer\n",
    "\n",
    "    # dims\n",
    "    mini_batch_size = hs[-1].shape[1]  # note -1 is dict key for initilized h state , not last\n",
    "    seq_length = len(xs)-1 # make sure this works every layer ?\n",
    "\n",
    "    vocab_size = params['Wxc'].shape[1]\n",
    "\n",
    "    # unpack parameters\n",
    "\n",
    "    Wxc = params['Wxc']\n",
    "    Wxu = params['Wxu']\n",
    "    Wxf = params['Wxf']\n",
    "    Wxo = params['Wxo']\n",
    "\n",
    "    Whc = params['Whc']\n",
    "    Whu = params['Whu']\n",
    "    Whf = params['Whf']\n",
    "    Who = params['Who']\n",
    "\n",
    "    bc = params['bc']\n",
    "    bu = params['bu']\n",
    "    bf = params['bf']\n",
    "    bo = params['bo']\n",
    "\n",
    "    #initialize gradients to zero\n",
    "\n",
    "    dWxc,dWxu,dWxf,dWxo = jnp.zeros_like(Wxc), jnp.zeros_like(Wxu), jnp.zeros_like(Wxf), jnp.zeros_like(Wxo)\n",
    "    dWhc,dWhu,dWhf,dWho = jnp.zeros_like(Whc), jnp.zeros_like(Whu), jnp.zeros_like(Whf), jnp.zeros_like(Who)\n",
    "    dbc,dbu,dbf,dbo = jnp.zeros_like(bc), jnp.zeros_like(bu), jnp.zeros_like(bf), jnp.zeros_like(bo)\n",
    "\n",
    "    # tmp variables to accumulate gradients over the backprop -- see differentiation graph\n",
    "    dhnext, dcnext = jnp.zeros_like(hs[0]), jnp.zeros_like(cs[0])\n",
    "\n",
    "    #need dictionary to pass dh derivative each t to earlier layer\n",
    "\n",
    "    dh_previous_layer = {}\n",
    "\n",
    "    #backward pass\n",
    "\n",
    "    for t in reversed(range(seq_length)):\n",
    "\n",
    "        dh = dh_next_layer[t] + dhnext # backprop into h\n",
    "\n",
    "        dc = jnp.multiply((1-cs[t]**2),jnp.multiply(gamma_os[t],dh) + dcnext) #backprop into c\n",
    "\n",
    "        dcnext = jnp.multiply(gamma_fs[t],dc)\n",
    "\n",
    "        dzc = jnp.multiply((1-c_tildes[t]**2),jnp.multiply(gamma_us[t],dc))  # backprop through tanh\n",
    "\n",
    "        dzu = jnp.multiply(gamma_us[t]*(1-gamma_us[t]),jnp.multiply(c_tildes[t],dc))  # sigmoid prime\n",
    "\n",
    "        dzf = jnp.multiply(gamma_fs[t]*(1-gamma_fs[t]),jnp.multiply(cs[t-1],dc))\n",
    "\n",
    "        dzo = jnp.multiply(gamma_os[t]*(1-gamma_os[t]),jnp.multiply(cs[t],dh))\n",
    "\n",
    "        dbc += jnp.sum(dzc,axis=1,keepdims=True)\n",
    "        dbu += jnp.sum(dzu,axis=1,keepdims=True)\n",
    "        dbf += jnp.sum(dzf,axis=1,keepdims=True)\n",
    "        dbo += jnp.sum(dzo,axis=1,keepdims=True)\n",
    "\n",
    "        dWhc += jnp.dot(dzc,hs[t-1].T)\n",
    "        dWhu += jnp.dot(dzu,hs[t-1].T)\n",
    "        dWhf += jnp.dot(dzf,hs[t-1].T)\n",
    "        dWho += jnp.dot(dzo,hs[t-1].T)\n",
    "\n",
    "        dWxc += jnp.dot(dzc,xs[t].T)\n",
    "        dWxu += jnp.dot(dzu,xs[t].T)\n",
    "        dWxf += jnp.dot(dzf,xs[t].T)\n",
    "        dWxo += jnp.dot(dzo,xs[t].T)\n",
    "\n",
    "        # four contributions to dhnext,one from each gate\n",
    "        dhnext = jnp.dot(Whc.T,dzc) + jnp.dot(Whu.T,dzu) + jnp.dot(Whf.T,dzf) + jnp.dot(Who.T,dzo)\n",
    "\n",
    "        dh_previous_layer[t] = jnp.dot(Wxc.T,dzc) + jnp.dot(Wxu.T,dzu) + jnp.dot(Wxf.T,dzf) + jnp.dot(Wxo.T,dzo)\n",
    "\n",
    "    grads = dict()\n",
    "\n",
    "    grads['Wxc']=dWxc\n",
    "    grads['Wxu']=dWxu\n",
    "    grads['Wxf']=dWxf\n",
    "    grads['Wxo']=dWxo\n",
    "\n",
    "    grads['Whc']=dWhc\n",
    "    grads['Whu']=dWhu\n",
    "    grads['Whf']=dWhf\n",
    "    grads['Who']=dWho\n",
    "\n",
    "    grads['bc']=dbc\n",
    "    grads['bu']=dbu\n",
    "    grads['bf']=dbf\n",
    "    grads['bo']=dbo\n",
    "\n",
    "    for parameter in grads.keys():\n",
    "        grads[parameter] = jnp.clip(grads[parameter], -5, 5) # clip to mitigate exploding gradients\n",
    "\n",
    "    return dh_previous_layer, grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IQM7lWq8fiHy",
    "outputId": "e61b5b2c-1b13-41ab-e50a-143b4bec2524"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DeviceArray([[ 7, 18,  6, 23, 15, 25, 13,  2],\n",
       "             [ 0, 17,  9,  3, 21, 12, 20,  5],\n",
       "             [22, 11, 19, 16,  1,  4, 14, 24],\n",
       "             [10,  8,  7, 18,  6, 23, 15, 25]], dtype=int32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cDQ5jkjQfkf-",
    "outputId": "bf8284ee-fb30-4351-ff02-ce232c4c01a2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(s_cache[1][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "aUh3M6K_rY3i",
    "outputId": "35d1b60e-8c20-4e82-8eab-5f8f3c5a589d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "print(len(all_params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "oOQBE04iWDkS",
    "outputId": "123b5ae4-6fc6-4f21-e2c1-fe88187308d5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "dict_keys(['Why', 'by'])\n"
     ]
    }
   ],
   "source": [
    "# test the loss_backward function\n",
    "dh_cache = {}\n",
    "\n",
    "dh_cache[2], all_grads[2] = loss_backward(targ,p_cache,s_cache[1][1],all_params[2])\n",
    "\n",
    "print(len(dh_cache[2]))\n",
    "print(all_grads[2].keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "aFjznK680ytU",
    "outputId": "79c47bb4-5231-4bf4-aec4-7bdafe4d6efb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'> 12\n",
      "dict_keys(['Wxc', 'Wxu', 'Wxf', 'Wxo', 'Whc', 'Whu', 'Whf', 'Who', 'bc', 'bu', 'bf', 'bo'])\n"
     ]
    }
   ],
   "source": [
    "#backprop lstm layer 1\n",
    "\n",
    "dh_cache[1],all_grads[1] = lstm_layer_backward(dh_cache[2],g_cache[1],s_cache[1],all_params[1])\n",
    "\n",
    "print(type(all_grads[1]),len(all_grads[1]))\n",
    "print(all_grads[1].keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OgB24gfyiGmo",
    "outputId": "e026361a-857f-40c1-e7a1-efeb612a3496"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'> 12\n",
      "dict_keys(['Wxc', 'Wxu', 'Wxf', 'Wxo', 'Whc', 'Whu', 'Whf', 'Who', 'bc', 'bu', 'bf', 'bo'])\n"
     ]
    }
   ],
   "source": [
    "#backprop lstm layer 2\n",
    "\n",
    "dh_cache[0],all_grads[0] = lstm_layer_backward(dh_cache[1],g_cache[0],s_cache[0],all_params[0])\n",
    "\n",
    "print(type(all_grads[0]),len(all_grads[0]))\n",
    "print(all_grads[0].keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "y4-IR4i6u4G5"
   },
   "outputs": [],
   "source": [
    "@jit  # decorator to jit compile the function for faster execution\n",
    "def sgd_step_adam(current_step,inputs,targets,h_inputs,c_inputs,all_params,all_grads_mems,all_sqrd_mems,beta1,beta2,learning_rate):\n",
    "    '''\n",
    "    Performs a single optimization step using the ADAM algorithm\n",
    "\n",
    "    Args:\n",
    "        current_step (int): counter of the steps performed so far during training\n",
    "        inputs (jax array): batch of input sequences shape=(mini_batch_size,seq_length)\n",
    "        targets (jax array): batch of target sequences shape=(mini_batch_size,seq_length)\n",
    "        h_inputs (dict): dictionary indexed by layer of initial hidden inputs, layers number 0,1,...,num_layers-1\n",
    "        c_inputs (dict): dict of intial cell state inputs each layer\n",
    "\n",
    "        all_params (dict): collection of weights, num_layers lstm layers + final dense loss layer\n",
    "\n",
    "                            all_params[0] - dictionary with 12 matrices for lstm layer 0\n",
    "                            all_params[1] - second lstm layer\n",
    "                            ...\n",
    "                            all_params[num_layers-1] - last lstm layer\n",
    "                            all_params[num_layers] - Why, by params of the output layer\n",
    "\n",
    "        all_grads (dict): running average of gradients each layer\n",
    "        all_sqrd_memes (dict): running average of squared gradients each layer\n",
    "\n",
    "        beta1 (float): ADAM parameter for gradient averaging\n",
    "        beta2 (float): ADAM parameter for squared gradient averaging\n",
    "        learning_rate (float): learing rate for gradient descent step\n",
    "    Returns:\n",
    "        loss (float): categorical cross entropy loss for this step\n",
    "        params_cache (tuple): shape = (params, grads, sqrd_grads) --> holds all the updated parameters\n",
    "                             after single ADAM step\n",
    "        hidden_cache (tuple): shape = (h_inputs,c_inputs) ---> final hidden states are saved to pass as inputs for next\n",
    "                                optimization step\n",
    "    '''\n",
    "    print('Adam Step Tracing...') # won't appear in jit compiled function after first call\n",
    "\n",
    "    # extract dimensions\n",
    "    mini_batch_size = inputs.shape[0]\n",
    "    seq_length = inputs.shape[1]\n",
    "\n",
    "    num_layers = len(all_params)-1  # this is number of lstm layers\n",
    "\n",
    "    vocab_size = all_params[0]['Wxc'].shape[1]\n",
    "\n",
    "    n = current_step\n",
    "\n",
    "    #one-hot encode the input batch\n",
    "\n",
    "    layer_inputs = {}\n",
    "\n",
    "    layer_inputs[0] = encode_inputs(inputs,vocab_size)\n",
    "\n",
    "    s_cache_dict,g_cache_dict = {},{}\n",
    "\n",
    "    dh_cache_dict = {}\n",
    "    all_grads_dict = {} # holds gradients for parameters each layer 0,1,..num_layers-1 = LSTM and num_layers = dense\n",
    "\n",
    "    h_inputs_next = {} # need to pass final hidden states to next sgd_step\n",
    "    c_inputs_next = {}\n",
    "\n",
    "    #forwad pass\n",
    "    for l in range(num_layers):\n",
    "\n",
    "        h = h_inputs[l]\n",
    "        c = c_inputs[l]\n",
    "\n",
    "        x = layer_inputs[l]\n",
    "\n",
    "        layer_params = all_params[l]\n",
    "\n",
    "        s_cache_dict[l], g_cache_dict[l] = lstm_forward(x,h,c,layer_params)\n",
    "\n",
    "        layer_inputs[l+1] = s_cache_dict[l][1]  # hidden states from this layer\n",
    "\n",
    "        #h_inputs_next[l]=jnp.mean(layer_inputs[l+1][seq_length-1],axis=1,keepdims=True)\n",
    "        #c_inputs_next[l]=jnp.mean(layer_inputs[l+1][seq_length-1],axis=1,keepdims=True)\n",
    "\n",
    "        h_inputs_next[l]=jnp.expand_dims(layer_inputs[l+1][seq_length-1][:,-1],axis=1)\n",
    "        c_inputs_next[l]=jnp.expand_dims(layer_inputs[l+1][seq_length-1][:,-1],axis=1)\n",
    "\n",
    "    #compute loss\n",
    "    loss,p_cache = loss_function(targets,layer_inputs[num_layers],all_params[num_layers])\n",
    "\n",
    "    #loss backward\n",
    "    dh_cache_dict[num_layers], all_grads_dict[num_layers] = loss_backward(targets,p_cache,s_cache_dict[num_layers-1][1],all_params[num_layers])\n",
    "\n",
    "    #lstm backwards\n",
    "    for l in reversed(range(num_layers)):\n",
    "\n",
    "        dh_cache_dict[l],all_grads_dict[l] = lstm_layer_backward(dh_cache_dict[l+1],g_cache_dict[l],s_cache_dict[l],all_params[l])\n",
    "\n",
    "    new_all_params = deepcopy(all_params)\n",
    "    new_all_grads_mems = deepcopy(all_grads_mems)\n",
    "    new_all_sqrd_mems = deepcopy(all_sqrd_mems)\n",
    "\n",
    "    #ADAM for all layers\n",
    "\n",
    "    #loop throuh lstm layers = 0,1,...num_layers-1\n",
    "\n",
    "    # dense layer = num_layers\n",
    "\n",
    "    for l in range(num_layers+1):\n",
    "\n",
    "        # perform parameter update with ADAM\n",
    "        for parameter in new_all_params[l].keys():\n",
    "\n",
    "            dparam = all_grads_dict[l][parameter] / mini_batch_size\n",
    "\n",
    "            new_all_grads_mems[l][parameter] = beta1*new_all_grads_mems[l][parameter] + (1-beta1)*dparam\n",
    "\n",
    "            new_all_sqrd_mems[l][parameter] = beta2*new_all_sqrd_mems[l][parameter] + (1-beta2)*dparam*dparam\n",
    "\n",
    "            grad_hat = new_all_grads_mems[l][parameter] / (1-beta1**(n+1))\n",
    "            sqrd_hat = new_all_sqrd_mems[l][parameter] / (1-beta2**(n+1))\n",
    "\n",
    "            new_all_params[l][parameter] += -learning_rate * grad_hat / (jnp.sqrt(sqrd_hat + 1e-8)) # ADAM update\n",
    "\n",
    "    params_cache = (new_all_params,new_all_grads_mems,new_all_sqrd_mems)\n",
    "\n",
    "    hidden_cache = (h_inputs_next,c_inputs_next)\n",
    "\n",
    "    return loss, params_cache, hidden_cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1EivFfmguSrl",
    "outputId": "a9f8b95a-d6db-4c58-d516-5d2deaec3db7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 3 3\n",
      "2 2\n",
      "Adam Step Tracing...\n"
     ]
    }
   ],
   "source": [
    "print(len(all_params),len(all_grads),len(all_sqrd))\n",
    "\n",
    "num_layers = len(all_params)-1\n",
    "\n",
    "h_inp = {}\n",
    "c_inp = {}\n",
    "\n",
    "for l in range(num_layers):\n",
    "    h_inp[l] = jnp.zeros((hidden_size,1))\n",
    "    c_inp[l] = jnp.zeros((hidden_size,1))\n",
    "\n",
    "print(len(h_inp),len(c_inp))\n",
    "\n",
    "test_loss, test_params_cache,test_hidden_cache = sgd_step_adam(\n",
    "                                                                    current_step=0,\n",
    "                                                                    inputs=inp,\n",
    "                                                                    targets=targ,\n",
    "                                                                    h_inputs=h_inp,\n",
    "                                                                    c_inputs=c_inp,\n",
    "                                                                    all_params=all_params,\n",
    "                                                                    all_grads_mems=all_grads,\n",
    "                                                                    all_sqrd_mems=all_sqrd,\n",
    "                                                                    beta1=0.9,\n",
    "                                                                    beta2=0.99,\n",
    "                                                                    learning_rate=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rr3FFPYRw-Dw",
    "outputId": "136c71de-df1c-45ec-e21a-3c213fa87933"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DeviceArray(26.064775, dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "cfHeUnxT-NxV"
   },
   "outputs": [],
   "source": [
    "def sample(seed_ix,n,key,h_inputs,c_inputs,all_params,temperature=1.0):\n",
    "    '''\n",
    "    Sample from the model starting from single character\n",
    "\n",
    "    Args:\n",
    "        seed_ix (int): index for starting character\n",
    "        n (int): number of characters to sample\n",
    "        key (PRNG key): key for jax random generation\n",
    "\n",
    "        h_inputs (dict): dict of hidden state inputs at each layer\n",
    "        c_inputs (dict): cell state inputs each layer\n",
    "\n",
    "        all_params (dict): collection of layer weights indexed by layer\n",
    "\n",
    "        temperature (dict): tune the softmax probabilities when sampling\n",
    "    Returns:\n",
    "        ixes (list): list of sampled character indices\n",
    "    '''\n",
    "\n",
    "    num_layers = len(all_params)-1\n",
    "\n",
    "    vocab_size = all_params[0]['Wxc'].shape[1]\n",
    "\n",
    "    #unpack params for output layer\n",
    "    Why = all_params[num_layers]['Why']\n",
    "    by = all_params[num_layers]['by']\n",
    "\n",
    "    x = jnp.zeros((vocab_size, 1))\n",
    "    x = x.at[seed_ix].set(1)\n",
    "\n",
    "    layer_inputs = {}\n",
    "\n",
    "    layer_inputs[0] = x\n",
    "\n",
    "    xs_layers = {}\n",
    "    hs_layers = {}\n",
    "    cs_layers = {}\n",
    "\n",
    "    for t in range(n+1):\n",
    "        xs_layers[t]={}\n",
    "        hs_layers[t]={}\n",
    "        cs_layers[t]={}\n",
    "\n",
    "    hs_layers[0]=h_inputs\n",
    "    cs_layers[0]=c_inputs\n",
    "\n",
    "    xs_layers[0][0] = x\n",
    "\n",
    "    ixes = [seed_ix]\n",
    "    for t in range(n):\n",
    "\n",
    "        for l in range(num_layers):\n",
    "\n",
    "            h = hs_layers[t][l]\n",
    "            c = cs_layers[t][l]\n",
    "\n",
    "            x = xs_layers[t][l]\n",
    "\n",
    "            Wxc = all_params[l]['Wxc']\n",
    "            Wxu = all_params[l]['Wxu']\n",
    "            Wxf = all_params[l]['Wxf']\n",
    "            Wxo = all_params[l]['Wxo']\n",
    "\n",
    "            Whc = all_params[l]['Whc']\n",
    "            Whu = all_params[l]['Whu']\n",
    "            Whf = all_params[l]['Whf']\n",
    "            Who = all_params[l]['Who']\n",
    "\n",
    "            bc = all_params[l]['bc']\n",
    "            bu = all_params[l]['bu']\n",
    "            bf = all_params[l]['bf']\n",
    "            bo = all_params[l]['bo']\n",
    "\n",
    "            zc = jnp.dot(Wxc,x) + jnp.dot(Whc,h) + bc  # linear activation for candidate cell state C~\n",
    "            zu = jnp.dot(Wxu,x) + jnp.dot(Whu,h) + bu  # linear activation for update gate\n",
    "            zf = jnp.dot(Wxf,x) + jnp.dot(Whf,h) + bf  # linear activation for forget gate\n",
    "            zo = jnp.dot(Wxo,x) + jnp.dot(Who,h) + bo  # linear activation for output gate\n",
    "\n",
    "            c_tilde = jnp.tanh(zc)\n",
    "\n",
    "            gamma_u = sigmoid(zu)\n",
    "            gamma_f = sigmoid(zf)\n",
    "            gamma_o = sigmoid(zo)\n",
    "\n",
    "            cs_layers[t+1][l] = jnp.tanh(jnp.multiply(c_tilde,gamma_u) + jnp.multiply(c,gamma_f))\n",
    "\n",
    "            hs_layers[t+1][l] = jnp.multiply(cs_layers[t+1][l],gamma_o) # hidden state\n",
    "\n",
    "            xs_layers[t][l+1] = hs_layers[t+1][l]\n",
    "\n",
    "        y = jnp.dot(Why,xs_layers[t][l+1]) + by\n",
    "\n",
    "        p = softmax(y/temperature)\n",
    "\n",
    "        key,subkey = random.split(key)  #use key to split, subkey for next random number\n",
    "\n",
    "        ix = random.choice(subkey,vocab_size,p=p.reshape(-1,))\n",
    "\n",
    "        x_new = jnp.zeros((vocab_size, 1))\n",
    "        x_new = x_new.at[ix].set(1)\n",
    "\n",
    "        #ixes.append(int(ix))\n",
    "        ixes.append(int(ix))\n",
    "\n",
    "        xs_layers[t+1][0] = x_new\n",
    "\n",
    "    return ixes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "cfHeUnxT-NxV"
   },
   "outputs": [],
   "source": [
    "@jit\n",
    "def validation_loss(inputs,targets,h_inputs,c_inputs,all_params):\n",
    "    '''\n",
    "    Forward pass to get loss on validation data\n",
    "\n",
    "    Args:\n",
    "        inputs (jax array): batch of sequences (mini_batch,seq_length)\n",
    "        targets (jax array): batch of sequences (mini_batch,seq_length)\n",
    "        h_inputs (dict): hidden state inputs each lstm layer\n",
    "        c_inputs (dict): cell state inputs each layer\n",
    "    Returns:\n",
    "        val_loss (float): cross entropy loss on batch of validation data\n",
    "    '''\n",
    "    print('Val loss tracing...') # won't appear in jit compiled function\n",
    "\n",
    "    num_layers = len(all_params)-1\n",
    "\n",
    "    vocab_size = all_params[0]['Wxc'].shape[1]\n",
    "\n",
    "    layer_inputs={}\n",
    "\n",
    "    layer_inputs[0] = encode_inputs(inputs,vocab_size)\n",
    "\n",
    "    for l in range(num_layers):\n",
    "\n",
    "        h = h_inputs[l]\n",
    "        c = c_inputs[l]\n",
    "\n",
    "        x = layer_inputs[l]\n",
    "\n",
    "        layer_params = all_params[l]\n",
    "\n",
    "        s_cache, _ = lstm_forward(x,h,c,layer_params)\n",
    "\n",
    "        layer_inputs[l+1]=s_cache[1] #hidden outputs this layer\n",
    "\n",
    "    val_loss,val_p_cache = loss_function(targets,layer_inputs[num_layers],all_params[num_layers])\n",
    "\n",
    "    return val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GD-UkVzDM9BJ",
    "outputId": "a4fe68a7-c3fa-488b-a3ba-9fd45da4b89f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val loss tracing...\n",
      "26.064775\n"
     ]
    }
   ],
   "source": [
    "h_inp = {}\n",
    "c_inp = {}\n",
    "\n",
    "for l in range(num_layers):\n",
    "    h_inp[l] = jnp.zeros((hidden_size,1))\n",
    "    c_inp[l] = jnp.zeros((hidden_size,1))\n",
    "\n",
    "test_val_loss = validation_loss(inp,targ,h_inp,c_inp,all_params)\n",
    "\n",
    "print(test_val_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "4tm0YeJQ0Cod"
   },
   "outputs": [],
   "source": [
    "def train_character_lstm(seq_length,\n",
    "                         hidden_sizes,\n",
    "                         mini_batch_size,\n",
    "                         learning_rate,\n",
    "                         total_steps,\n",
    "                         steps_sample_freq,\n",
    "                         key,\n",
    "                         training_data,\n",
    "                         validation_data=None,\n",
    "                         beta1=0.9,\n",
    "                         beta2=0.999):\n",
    "    '''\n",
    "    Constructs and trains the lstm network on a body of text\n",
    "\n",
    "    Args:\n",
    "        seq_length (int): number of character per training sequence\n",
    "        hidden_sizes (list): list of layer sizes for num_layers=len(hidden_sizes) LSTM layers\n",
    "        mini_batch_size (int): size of batches for each training step\n",
    "        learning_rate (float): learning rate for gradient descent\n",
    "\n",
    "        total_steps (int): number of ADAM steps to perform\n",
    "        steps_sample_freq (int): how often to sample text from the model / record training/valdiation losses\n",
    "\n",
    "        key (PRNG key): key for seeding jax random functions\n",
    "\n",
    "        training_data (str): string of text data to train the model on\n",
    "        validation_data (str): string of optional validation text data\n",
    "\n",
    "        beta1 (float): hyperparameter for ADAM , running gradient average\n",
    "        beta2 (float): hyperparamter for ADAM, running second moment average\n",
    "\n",
    "    Returns:\n",
    "        history (tuple): losses for the training cycle, sampled with steps_sample_freq\n",
    "\n",
    "                         step_list - current training step\n",
    "                         smooth_loss - exponentially smoothed training loss\n",
    "                         train_loss - training loss\n",
    "                         val_loss - loss on validation data\n",
    "        params_cache (tuple): cache with all parameters / gradient memories\n",
    "        hidden_cache (tuple): cache with final hidden states output\n",
    "    '''\n",
    "\n",
    "    #unique characters in the data set\n",
    "    chars = set(list(training_data))\n",
    "    vocab_size = len(chars)\n",
    "\n",
    "    #character encoding\n",
    "    char_to_ix = {ch:i for i,ch in enumerate(chars)}\n",
    "    ix_to_char = {i:ch for i,ch in enumerate(chars)}\n",
    "\n",
    "    # initialize data generators\n",
    "    training_generator = get_mini_batch(mini_batch_size,seq_length,char_to_ix,training_data)\n",
    "    if validation_data:\n",
    "       validation_generator = get_mini_batch(mini_batch_size,seq_length,char_to_ix,validation_data)\n",
    "\n",
    "    #current training step counter\n",
    "    n = 0\n",
    "\n",
    "    num_layers = len(hidden_sizes)   # number of lstm layers\n",
    "\n",
    "    #initialize the model weight matrices\n",
    "\n",
    "    all_params = {}\n",
    "    all_grads = {}\n",
    "    all_sqrd = {}\n",
    "\n",
    "    h_inputs,c_inputs = {},{}\n",
    "\n",
    "    h_inputs_val,c_inputs_val={},{} # start val network with blank states each time\n",
    "\n",
    "    for l in range(num_layers):\n",
    "\n",
    "        if l==0:\n",
    "           prev_layer_size = vocab_size\n",
    "        elif l>0:\n",
    "           prev_layer_size = hidden_sizes[l-1]\n",
    "\n",
    "        key,subkey = random.split(key)\n",
    "\n",
    "        all_params[l],all_grads[l],all_sqrd[l] = initialize_lstm_weights(subkey,hidden_sizes[l],prev_layer_size)\n",
    "\n",
    "        h_inputs[l] = jnp.zeros((hidden_sizes[l],1))\n",
    "        c_inputs[l] = jnp.zeros((hidden_sizes[l],1))\n",
    "\n",
    "        h_inputs_val[l] = jnp.zeros((hidden_sizes[l],1))\n",
    "        c_inputs_val[l] = jnp.zeros((hidden_sizes[l],1))\n",
    "\n",
    "    all_params[num_layers],all_grads[num_layers],all_sqrd[num_layers] = initialize_dense_weights(key,hidden_sizes[num_layers-1],vocab_size)\n",
    "\n",
    "    total_params = 0\n",
    "\n",
    "    for k1,v1 in all_params.items():\n",
    "        layer_total = 0\n",
    "        for k2,v2 in v1.items():\n",
    "            layer_total += v2.size\n",
    "            print(k1,k2,v2.shape)\n",
    "\n",
    "        total_params += layer_total\n",
    "        print(f'Layer {k1}: {layer_total} parameters')\n",
    "\n",
    "    print(f'LSTM Model has {num_layers+1} layers with {total_params} parameters')\n",
    "\n",
    "    #keep list of loss each training step\n",
    "    step_list = []\n",
    "\n",
    "    tau = len(training_data) / (mini_batch_size * seq_length)\n",
    "    alpha = jnp.exp(-4./tau).item()\n",
    "    #print(f'tau: {tau} alpha: {alpha}')\n",
    "\n",
    "    smooth_loss = -jnp.log(1.0/vocab_size)*seq_length\n",
    "\n",
    "    train_losses,smooth_losses,val_losses = [],[],[]\n",
    "\n",
    "    last_epoch = 0\n",
    "\n",
    "    sample_size = 200\n",
    "\n",
    "    sample_ix = [0]*sample_size\n",
    "\n",
    "    while n < total_steps:\n",
    "\n",
    "          inputs,targets,current_epoch = next(training_generator)\n",
    "\n",
    "          if current_epoch > last_epoch:\n",
    "\n",
    "             last_epoch = current_epoch\n",
    "\n",
    "             for l in range(num_layers):\n",
    "                 h_inputs[l] = jnp.zeros((hidden_sizes[l],1))\n",
    "                 c_inputs[l] = jnp.zeros((hidden_sizes[l],1))\n",
    "\n",
    "          current_loss, params_cache, hidden_cache = sgd_step_adam(\n",
    "                                                                current_step=n,\n",
    "                                                                inputs=inputs,\n",
    "                                                                targets=targets,\n",
    "                                                                h_inputs=h_inputs,\n",
    "                                                                c_inputs=c_inputs,\n",
    "                                                                all_params=all_params,\n",
    "                                                                all_grads_mems=all_grads,\n",
    "                                                                all_sqrd_mems=all_sqrd,\n",
    "                                                                beta1=beta1,\n",
    "                                                                beta2=beta2,\n",
    "                                                                learning_rate=learning_rate)\n",
    "\n",
    "          smooth_loss = alpha*smooth_loss + (1-alpha)*current_loss\n",
    "\n",
    "          # need to unpack caches so they are passed to next step\n",
    "          h_inputs,c_inputs = hidden_cache\n",
    "          all_params,all_grads,all_sqrd=params_cache\n",
    "\n",
    "          # sample from the model now and then\n",
    "          if n % steps_sample_freq == 0:\n",
    "             key,subkey=random.split(key) #key to split, subkey to gen next random sample\n",
    "\n",
    "             sample_ix = sample(sample_ix[-1],sample_size,subkey,h_inputs_val,c_inputs_val,all_params)\n",
    "\n",
    "             txt = ''.join(ix_to_char[int(ix)] for ix in sample_ix)\n",
    "             txt_wrap = wrap(txt,80)\n",
    "             txt_wrap = [line.center(100) for line in txt_wrap]\n",
    "             txt = '\\n'.join(txt_wrap)  # \\n aren't in the character set so wrap text to make readable\n",
    "\n",
    "             print('----\\n %s \\n----' % (txt,))\n",
    "\n",
    "             print(f'Step n: {n}\\t Epoch: {current_epoch}')\n",
    "\n",
    "             print(f'Train Current: {current_loss:.4f}\\tTrain Smoothed: {smooth_loss:.4f}')\n",
    "\n",
    "             step_list.append(n)\n",
    "             train_losses.append(current_loss)\n",
    "             smooth_losses.append(smooth_loss)\n",
    "\n",
    "             #compute_validation_loss\n",
    "             if validation_generator:\n",
    "                inputs_val,targets_val,val_epoch = next(validation_generator)\n",
    "\n",
    "                val_loss = validation_loss(inputs_val,targets_val,h_inputs_val,c_inputs_val,all_params)\n",
    "\n",
    "                print(f'Val Current: {val_loss:.4f}')\n",
    "\n",
    "                val_losses.append(val_loss)\n",
    "\n",
    "          n+=1\n",
    "\n",
    "    history = (step_list,smooth_losses,train_losses,val_losses)\n",
    "\n",
    "    return history, params_cache, hidden_cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zqn98JR26NIi",
    "outputId": "06c8c12c-42e2-4ce7-fc5d-ca71eea0e16c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Wxc (8, 26)\n",
      "0 Wxu (8, 26)\n",
      "0 Wxf (8, 26)\n",
      "0 Wxo (8, 26)\n",
      "0 bc (8, 1)\n",
      "0 bu (8, 1)\n",
      "0 bf (8, 1)\n",
      "0 bo (8, 1)\n",
      "0 Whc (8, 8)\n",
      "0 Whu (8, 8)\n",
      "0 Whf (8, 8)\n",
      "0 Who (8, 8)\n",
      "Layer 0: 1120 parameters\n",
      "1 Wxc (8, 8)\n",
      "1 Wxu (8, 8)\n",
      "1 Wxf (8, 8)\n",
      "1 Wxo (8, 8)\n",
      "1 bc (8, 1)\n",
      "1 bu (8, 1)\n",
      "1 bf (8, 1)\n",
      "1 bo (8, 1)\n",
      "1 Whc (8, 8)\n",
      "1 Whu (8, 8)\n",
      "1 Whf (8, 8)\n",
      "1 Who (8, 8)\n",
      "Layer 1: 544 parameters\n",
      "2 Why (26, 8)\n",
      "2 by (26, 1)\n",
      "Layer 2: 234 parameters\n",
      "LSTM Model has 3 layers with 1898 parameters\n",
      "Adam Step Tracing...\n",
      "----\n",
      "           jhlchjxsngzplyevzuhpdcgdqbnzgjspcseurpggvnxwetkulchxinenlgnixlohkfylzrqsfzxnpshc          \n",
      "          yxrvusowimzqcpxshelqwqjytijcnwtpjsnjlssltinfbfclkuybbqvlvpnalaovifxgpqnizrmzayfq          \n",
      "                             vdwuwufdezbwoxwbkikimfndevmrorugdudtwpsqj                               \n",
      "----\n",
      "Step n: 0\t Epoch: 0\n",
      "Train Current: 32.5810\tTrain Smoothed: 32.5810\n",
      "Val loss tracing...\n",
      "Val Current: 32.5795\n",
      "----\n",
      "           jhijlmnoppqrssuwxyabddjhjmnnoppqsqvuwxyzacabcdghhiijlmmnopqstvvtvvuvvvuvvuvwxyza          \n",
      "          ccgghjmnnopqrsrrstvwxyzabbhjklmnoppqssvuxyzacdefghgiikllmopqrswxybabcefgikikmmop          \n",
      "                             qrrqssvuvvutvvwxyzabcdefgihjklnopqstuvutw                               \n",
      "----\n",
      "Step n: 250\t Epoch: 0\n",
      "Train Current: 5.2679\tTrain Smoothed: 23.2470\n",
      "Val Current: 5.5064\n",
      "----\n",
      "           wwxyzatefghijklmopqrstuvpqrstuvwxyzabcdefghijkopqrstuvwxyzabcdefghijklmopqrstuvw          \n",
      "          xyzabcdefghijklmnopqrstuwxyzabcdefghiijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy          \n",
      "                             zabcdefghijklmnopqrstuvwxyzabdfghijklmnop                               \n",
      "----\n",
      "Step n: 500\t Epoch: 0\n",
      "Train Current: 0.5523\tTrain Smoothed: 11.5570\n",
      "Val Current: 0.8083\n",
      "----\n",
      "           pqrstuvwxyxyzabcdeghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnop          \n",
      "          qrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqr          \n",
      "                             stuvwxyzabcdefghijklmnopqrstuvwxyzabcdefg                               \n",
      "----\n",
      "Step n: 750\t Epoch: 0\n",
      "Train Current: 0.1724\tTrain Smoothed: 5.4921\n",
      "Val Current: 0.3986\n",
      "----\n",
      "           ghijklmnnopqrstuvrvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcde          \n",
      "          fghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefg          \n",
      "                             hijklmnopqpqrstuvwxyzabcdefghijklmnopqrst                               \n",
      "----\n",
      "Step n: 1000\t Epoch: 0\n",
      "Train Current: 0.1059\tTrain Smoothed: 2.6129\n",
      "Val Current: 0.3142\n",
      "----\n",
      "           tuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstu          \n",
      "          vwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvw          \n",
      "                             xyzabcdefghijklmnopqpqrstuvwxyzabcdefghij                               \n",
      "----\n",
      "Step n: 1250\t Epoch: 0\n",
      "Train Current: 0.0765\tTrain Smoothed: 1.2543\n",
      "Val Current: 0.2428\n",
      "----\n",
      "           jnjklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghi          \n",
      "          jklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk          \n",
      "                             lmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz                               \n",
      "----\n",
      "Step n: 1500\t Epoch: 1\n",
      "Train Current: 0.0483\tTrain Smoothed: 0.6126\n",
      "Val Current: 0.0893\n",
      "----\n",
      "           zabcdefghijklmnopqrstuvwxyzabcyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuv          \n",
      "          wxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvvw          \n",
      "                             xyzabcdefghijklmnopqrstuvwxyzabcdefghijkl                               \n",
      "----\n",
      "Step n: 1750\t Epoch: 1\n",
      "Train Current: 0.0418\tTrain Smoothed: 0.3071\n",
      "Val Current: 0.0728\n",
      "----\n",
      "           lmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklm          \n",
      "          nopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmno          \n",
      "                             pqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcd                               \n",
      "----\n",
      "Step n: 2000\t Epoch: 1\n",
      "Train Current: 0.0295\tTrain Smoothed: 0.1602\n",
      "Val Current: 0.0695\n",
      "----\n",
      "           dfghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdef          \n",
      "          ghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefgh          \n",
      "                             ijklmnopqrstuvwxyzabcdefghijklmnopqrstuvw                               \n",
      "----\n",
      "Step n: 2250\t Epoch: 1\n",
      "Train Current: 0.0235\tTrain Smoothed: 0.0884\n",
      "Val Current: 0.0495\n",
      "----\n",
      "           wxyzabcdefghijkllmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvw          \n",
      "          xyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy          \n",
      "                             zabcdefghijklmnopqrstuvwxyzabcdefghijklmn                               \n",
      "----\n",
      "Step n: 2500\t Epoch: 1\n",
      "Train Current: 0.0184\tTrain Smoothed: 0.0523\n",
      "Val Current: 0.0410\n",
      "----\n",
      "           nopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmno          \n",
      "          pqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopq          \n",
      "                             rstuvwxyzabcdefghijklmnopqrstuvwxyzabcdef                               \n",
      "----\n",
      "Step n: 2750\t Epoch: 2\n",
      "Train Current: 0.0150\tTrain Smoothed: 0.0336\n",
      "Val Current: 0.0202\n",
      "----\n",
      "           fghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefg          \n",
      "          hijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghi          \n",
      "                             jklmnopqrstuvwxyzabcdefghijklmnopqrstuvwx                               \n",
      "----\n",
      "Step n: 3000\t Epoch: 2\n",
      "Train Current: 0.0125\tTrain Smoothed: 0.0232\n",
      "Val Current: 0.0199\n",
      "----\n",
      "           xyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy          \n",
      "          zabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyza          \n",
      "                             bcdefghijklmnopqrstuvwxyzabcdefghijklmnop                               \n",
      "----\n",
      "Step n: 3250\t Epoch: 2\n",
      "Train Current: 0.0107\tTrain Smoothed: 0.0171\n",
      "Val Current: 0.0164\n",
      "----\n",
      "           pqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkmnopqrstuvwxyzabcdefghijklmnopqr          \n",
      "          stuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrst          \n",
      "                             uvwxyzabcdefghijklmnopqrstuvwxyzabcdefghi                               \n",
      "----\n",
      "Step n: 3500\t Epoch: 2\n",
      "Train Current: 0.0109\tTrain Smoothed: 0.0132\n",
      "Val Current: 0.0134\n",
      "----\n",
      "           ijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghij          \n",
      "          klmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkl          \n",
      "                             mnopqrstuvwxyzabcdefghijklmnopqrstuvwxyza                               \n",
      "----\n",
      "Step n: 3750\t Epoch: 2\n",
      "Train Current: 0.0071\tTrain Smoothed: 0.0105\n",
      "Val Current: 0.0117\n"
     ]
    }
   ],
   "source": [
    "mykey = random.PRNGKey(1)\n",
    "\n",
    "seq_length = 10\n",
    "\n",
    "hidden_sizes = [8,8]\n",
    "\n",
    "mini_batch_size = 16\n",
    "\n",
    "train_data = alphabet_data[:-1012]\n",
    "val_data = alphabet_data[1012:]\n",
    "\n",
    "history,out_params,out_hidden = train_character_lstm(                  \n",
    "                                  seq_length=seq_length,\n",
    "                                  hidden_sizes=hidden_sizes,\n",
    "                                  mini_batch_size=mini_batch_size,\n",
    "                                  learning_rate=0.01,\n",
    "                                  total_steps=4000,\n",
    "                                  steps_sample_freq=250,\n",
    "                                  key=mykey,\n",
    "                                  training_data=train_data,\n",
    "                                  validation_data=val_data,\n",
    "                                  beta1=0.9,\n",
    "                                  beta2=0.999)                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "8u4x8mh3YfIZ"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "def plot_training_history(history,mini_batch_size,seq_length,training_length,plot_smooth=False,plot_val=True):\n",
    "\n",
    "    step_list,smooth_losses,train_losses,val_losses = history\n",
    "\n",
    "    fig,ax = plt.subplots(figsize=(10,6))\n",
    "\n",
    "    t_list = np.array(step_list)\n",
    "\n",
    "    t_list = t_list * mini_batch_size * seq_length / training_length\n",
    "\n",
    "    train_log_loss = np.array(train_losses)\n",
    "\n",
    "    if plot_smooth:\n",
    "       smooth_log_loss = np.array(smooth_losses)\n",
    "       ax.plot(t_list,smooth_log_loss,'bo-')\n",
    "\n",
    "    ax.plot(t_list,train_log_loss,'go-')\n",
    "\n",
    "    if len(val_losses)==len(step_list) and plot_val:\n",
    "       val_log_loss = np.array(val_losses)\n",
    "\n",
    "       ax.plot(t_list,val_log_loss,'ro-')\n",
    "\n",
    "    ax.set_ylabel('$L$')\n",
    "    ax.set_xlabel('Epochs')\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "id": "-IZoPwacZcd6",
    "outputId": "54983f54-2f79-4835-a2a6-3ff05e84d9cb"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAINCAYAAAAJGy/3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABja0lEQVR4nO3deXwU9eH/8ffkTiAJhNwkJFzhFAURQUXxQrBSEdFWUaHaw4pWvn4t1dqv4rdWWrWo/an0az1qi7eAUq+KFfBEQUHuO4EQEm5yQc6d3x9DQkJustnPHq/n4zGPnZ2dnX1nl9W8MzOfsWzbtgUAAAAAaFKQ6QAAAAAA4O0oTgAAAADQAooTAAAAALSA4gQAAAAALaA4AQAAAEALKE4AAAAA0AKKEwAAAAC0gOIEAAAAAC0IMR2go7lcLu3Zs0fR0dGyLMt0HAAAAACG2Lat4uJipaamKiiobfuQ/L447dmzR+np6aZjAAAAAPASubm5SktLa9Nz/L44RUdHS3LenJiYGMNpAAAAAJhSVFSk9PT02o7QFn5fnGoOz4uJiaE4AQAAADilU3gYHAIAAAAAWkBxAgAAAIAWUJwAAAAAoAUUJwAAAABoAcUJAAAAAFpAcQIAAACAFlCcAAAAAKAFFCcAAAAAaAHFCQAAAABaQHECAAAAgBZQnAAAAACgBRQnAAAAAGgBxQkAAAAAWhBiOkCgqCir0KLHnlFJznZ1zuytH959m8IiwkzHAgAAANAKFCcP+MftM3XhS3M0uaS6dlnun+7Wkql36aanHjGYDAAAAEBrcKheB/vH7TN1w9OPqnud0iRJ3UuqdcPTj+oft880lAwAAABAa1GcOlBFWYUufGmOpIZvdM39Mf+Yo4qyCo/mAgAAANA2FKcOtOixZ5ReUt3kmxwkqUdxtRY99ownYwEAAABoI4pTByrJ2e7W9QAAAACYQXHqQJ0ze7t1PQAAAABmWLZt26ZDdKSioiLFxsaqsLBQMTExHn3tirIK7U2IUvcmDtdzSdodHazkfUcZmhwAAADoYO3pBuxx6kBhEWFaMvUuSU5Jqqvm/tKb7qI0AQAAAF6O4tTBbnrqEc2b/mvldQ6ut3x352A9+eNfcx0nAAAAwAdQnDzgpqceUdL+o3rjvodqlw2vWq3vwylNAAAAgC+gOHlIWESYrn3oPuXHOm95z84r9MYbUmGh4WAAAAAAWkRx8rB9SZ0lSWelrtKxY9KrrxoOBAAAAKBFFCcPK0pLkCSdnbRRkvTccybTAAAAAGgNipOHVWX2kCT1qNylsDDp22+lVasMhwIAAADQLIqTh4X27S9J6pK/V1dd5Sx7/nmDgQAAAAC0iOLkYTEDzpAkJewt0S23OMvmzZOOHTOXCQAAAEDzKE4eljTkHElScmG1Ro8sUkaGM7Le/PmGgwEAAABoEsXJwxIzBqo4TAqypb3rv6rd68QgEQAAAID3ojh5mBUUpD0JEZKkg+tXaNo0KShIWrZM2rrVbDYAAAAAjaM4GXAotYsk6eimtUpPl8aNc5a/8IK5TAAAAACaRnEyoCw9VZJkb98uSfrpT53lf/+7VFlpKBQAAACAJlGcTOjdW5IUsStPknTFFVJiolRQIL3/vslgAAAAABpDcTIgqt9gSVLX/COSpNBQado05zEGiQAAAAC8D8XJgG6DR0iSUveXyXa5JKl2dL3335fy8kwlAwAAANAYipMB3QePUrUlRVVKh7M3SpKysqTzz5dcLudcJwAAAADeg+JkQGSnWOV1DZYk7V3zZe3ymkEinn/eKVAAAAAAvAPFyZD9SZ0lSUUbV9cuu/pqKSZGys6WliwxFAwAAABAAxQnQ0q6J0qSKrZuql0WFSVNmeLMM0gEAAAA4D0oToZU9cyQJIXk7Ky3vOZwvQULpIMHPZ0KAAAAQGMoToaE9u0vSYrevb/e8mHDpKFDpYoKad48E8kAAAAAnIziZEjswKGSpKS9JQ0eq9nr9Nxzkm17MhUAAACAxlCcDEkeco4kKaHYpYqiw/Ueu/56KSJCWrdOWrHCRDoAAAAAdVGcDElM66dDkc58QZ0hySWpSxfpmmuceQaJAAAAAMwzWpzmzp2rIUOGKCYmRjExMRo1apQ++OCD2sdt29asWbOUmpqqyMhIjRkzRuvXrzeY2H0sy1J+QoQk6eC6hruVag7Xe/VVqaTh0XwAAAAAPMhocUpLS9Mf//hHrVy5UitXrtRFF12kK6+8srYcPfLII5ozZ46eeuoprVixQsnJybr00ktVXFxsMrbbHE6NkyQd27yuwWOjR0t9+zql6Y03PJ0MAAAAQF1Gi9OECRN0+eWXKysrS1lZWfrDH/6gzp07a/ny5bJtW0888YTuu+8+TZo0SYMHD9ZLL72ko0eP6pVXXjEZ223KeqRKkuzt2xs8Zln1B4kAAAAAYI7XnONUXV2t1157TaWlpRo1apSys7NVUFCgsWPH1q4THh6uCy64QF9++WWT2ykvL1dRUVG9yVsF9eojSYrMzW/08ZtukkJCpK++kvzkCEUAAADAJxkvTmvXrlXnzp0VHh6uW2+9VQsXLtTAgQNVUFAgSUpKSqq3flJSUu1jjZk9e7ZiY2Nrp/T09A7N3x5R/U+TJMXlH2n08eRk6YornPnnn/dQKAAAAAANGC9O/fr10+rVq7V8+XL98pe/1NSpU7Vhw4baxy3Lqre+bdsNltV17733qrCwsHbKzc3tsOztFT94hCQp5UC57KqqRtepOVzvH/+Qyss9lQwAAABAXcaLU1hYmPr06aPhw4dr9uzZOv300/Xkk08qOTlZkhrsXdq3b1+DvVB1hYeH147SVzN5q+6DRqoySAqvlo5sb/xYvMsuk7p3lw4elN55x8MBAQAAAEjyguJ0Mtu2VV5erp49eyo5OVmLFy+ufayiokLLli3TOeecYzCh+0RGdNbuuGBJ0t41XzW6TkiI9JOfOPMMEgEAAACYYbQ4/fa3v9Vnn32mnJwcrV27Vvfdd5+WLl2qKVOmyLIszZgxQw8//LAWLlyodevWadq0aYqKitL1119vMrZb7Ut29ogVbVzV5Do33+zcfvyxlJPjgVAAAAAA6gkx+eJ79+7VjTfeqPz8fMXGxmrIkCH68MMPdemll0qSZs6cqWPHjum2227T4cOHdfbZZ+ujjz5SdHS0ydhuVZKWKK07rMptm5tcp2dP6ZJLnOL04ovSgw96MCAAAAAAWbZt26ZDdKSioiLFxsaqsLDQK893+ui2yzR27kf65ryeGvHZjibXe/116cc/ltLSnL1OwcGeywgAAAD4g/Z0A687xynQhPcdIEmKyTvQ7HoTJ0pxcdLu3dJHH3kgGAAAAIBaFCfDYgcNkyQl7ittdr3wcOeCuBKDRAAAAACeRnEyLOU0Z4TAuFKXKg7tb3bdW25xbhctkvbu7ehkAAAAAGpQnAxLTO6t/Z2c+YI1Xza77uDB0siRUlWVc0FcAAAAAJ5BcTLMsiztSYyUJB1et7LF9Wv2Oj33nOTfw3oAAAAA3oPi5AWOpMZJko5uXtfiuj/6kdSpk7Rli/T55x2dDAAAAIBEcfIKZT3SJEnWjqaHI68RHe0MSy4xSAQAAADgKRQnLxDUu48kKTK3oFXr//Snzu2bb0qFhR2VCgAAAEANipMX6DTgNElSt/wjrVr/7LOlQYOkY8ekV1/twGAAAAAAJFGcvELC4LMlScmHKmRXVLS4vmWd2OvE4XoAAABAx6M4eYH0/iN0LEQKcUlHtq5t1XNuuEEKC5O+/VZataqDAwIAAAABjuLkBSLCopTbLUSStG/NV616Tny8dNVVzvzzz3dUMgAAAAASxclrHEiKkSQVbVzd6ufUHK43b55zvhMAAACAjkFx8hIl6UmSpKptW1r9nIsukjIznZH15s/voGAAAAAAKE7ewtUzQ5IUmpPb6ucEBUm33OLMM0gEAAAA0HEoTl4iPGugJCk270CbnjdtmlOgli2TtrR+ZxUAAACANqA4eYkuA4dJkpL2lUq23ernpaVJ48Y58y+80BHJAAAAAFCcvETqkHMlSTFltir25bfpuTWDRPz971JlpZuDAQAAAKA4eYvE+AztibYkSXtbOSR5jSuukBITpb17pffe64h0AAAAQGCjOHkJy7KUnxQpSTq0fmWbnhsa6pzrJHFNJwAAAKAjUJy8yJHUbpKkss3r2/zcmtH13n9fystzZyoAAAAAFCcvUtGjuyTJys5u83OzsqTzz5dcLudcJwAAAADuQ3HyIkF9+kqSonILTun5NYNEPP+8U6AAAAAAuAfFyYt07j9EktQtv/CUnn/11VJsrJSdLS1Z4s5kAAAAQGCjOHmRxCGjJElJRypll5W1+flRUdKUKc78c8+5MxkAAAAQ2ChOXiS9zzCVhEpBtnRk0+pT2kbN4XoLFkgHD7ovGwAAABDIKE5eJCI0UrviQyRJ+9p4LacaQ4dKw4ZJFRXSvHnuTAcAAAAELoqTlzmQHCtJKtm05pS3UTM0+XPPSbbtjlQAAABAYKM4eZnSHkmSpKrtW095G9dfL0VESOvWSd98465kAAAAQOCiOHkZu2dPSVJYTu4pb6NLF+maa5x5BokAAAAA2o/i5GXCswZKkmL3tG9kh5pBIl57TSopaW8qAAAAILBRnLxM3KDhkqTkfUfbdYLS6NFS375OaXrjDXelAwAAAAITxcnLpA4epWpLiqqwVbHn1A/Xs6wTe504XA8AAABoH4qTl0nsmqbdsZYkae/3X7ZrWzfdJIWESF99Ja1f7450AAAAQGCiOHkZy7JUkBglSTq84dt2bSs5WZowwZl//vn2JgMAAAACF8XJCxV27yZJKt+8od3bqjlc7x//kMrL2705AAAAICBRnLxQRUaaJCkoO6fd27rsMql7d+ngQemdd9q9OQAAACAgUZy8UHCfLElS1O697d9WsHTzzc48g0QAAAAAp4bi5IViBpwhSYrPL3TL9n7yE+d28WIpO9stmwQAAAACCsXJCyWcdrZzW1Qlu7S03dvr2VO65BJn/sUX2705AAAAIOBQnLxQj55n6HCEM39kw3du2WbNIBEvvihVV7tlkwAAAEDAoDh5oYiQCOXGh0qS9q/72i3bnDhRiouTdu+WPvrILZsEAAAAAgbFyUsdSI6VJJVsXOOW7YWHOxfElRgkAgAAAGgripOXOpaeLElybd/qtm3ecotzu2iRtLf9A/YBAAAAAYPi5KVcvXtKksJ37nbbNgcPlkaOlKqqnAviAgAAAGgdipOXiswaJEmKzTvk1u3WDBLx3HOSbbt10wAAAIDfojh5qbhBZ0mSkvYfdesweD/6kdS5s7Rli/T5527bLAAAAODXKE5eqvvAs1UZJIVXSxW5OW7bbufO0o9/7MwzSAQAAADQOhQnL5UYm6qdXS1J0r41y9267ZrD9d58UzpyxK2bBgAAAPwSxclLWZalvYmdJElH1n/r1m2PGCENGiQdOya9+qpbNw0AAAD4JYqTFyvq3k2SVL51o1u3a1n1B4kAAAAA0DyKkxeryOghSQrKznH7tm+4QQoLk777Tlq1yu2bBwAAAPwKxcmLhfbNkiR12u3+q9XGx0tXXeXMP/+82zcPAAAA+BWjxWn27Nk666yzFB0drcTERE2cOFGbN2+ut860adNkWVa9aeTIkYYSe1b0gDMkSQn5RR2y/ZrD9ebNc853AgAAANA4o8Vp2bJlmj59upYvX67FixerqqpKY8eOVWlpab31xo0bp/z8/Nrp/fffN5TYs5KGjJIkdS2tlt0Bw99ddJGUmSkVFkrz57t98wAAAIDfMFqcPvzwQ02bNk2DBg3S6aefrhdffFG7du3St9/WH0UuPDxcycnJtVNcXJyhxJ7VI22Q9kU580c2fOf27QcFSbfc4swzSAQAAADQNK86x6mwsFCSGhSjpUuXKjExUVlZWfrZz36mffv2NbmN8vJyFRUV1Zt8VURIhHbHh0qSDqz7pkNeY9o0p0AtWyZt2dIhLwEAAAD4PK8pTrZt66677tJ5552nwYMH1y4fP368Xn75ZX3yySf685//rBUrVuiiiy5SeXl5o9uZPXu2YmNja6f09HRP/Qgd4mBqF0lS6ea1HbL9tDRp/Hhn/oUXOuQlAAAAAJ9n2bZtmw4hSdOnT9d7772nzz//XGlpaU2ul5+fr4yMDL322muaNGlSg8fLy8vrlaqioiKlp6ersLBQMTExHZK9Iy269nT98M01+u6HZ2nYOx2z1+ntt50R9pKSpNxcKTS0Q14GAAAAMKqoqEixsbGn1A28Yo/THXfcoUWLFmnJkiXNliZJSklJUUZGhrZu3dro4+Hh4YqJiak3+bRevSRJ4bvyOuwlfvADpzTt3Su9916HvQwAAADgs4wWJ9u2dfvtt2vBggX65JNP1LNnzxafc/DgQeXm5iolJcUDCc2L7Occttg171CHvUZoqDR1qjPPIBEAAABAQ0aL0/Tp0zVv3jy98sorio6OVkFBgQoKCnTs+EWFSkpKdPfdd+urr75STk6Oli5dqgkTJig+Pl5X1Vy91c/FDRouSUo8WCZVVXXY69SMrvfBB1Jex+3cAgAAAHyS0eI0d+5cFRYWasyYMUpJSamdXn/9dUlScHCw1q5dqyuvvFJZWVmaOnWqsrKy9NVXXyk6OtpkdI9J6z9CZcFSiEuqyN7WYa+TlSWdf77kckl//3uHvQwAAADgk7xmcIiO0p4TwLyBbdvanBSs/vtt7Z7/otImTeuw1/rnP6WbbpJ69pS2bXOGKQcAAAD8hc8PDoGmWZalvUmdJUmF691/Edy6rr5aio2VsrOlJUs69KUAAAAAn0Jx8gHF3eMlSRVbN3Xo60RFSVOmOPMMEgEAAACcQHHyAZU9e0iSgnJ2dvhr/fSnzu2CBdLBgx3+cgAAAIBPoDj5gNA+/SRJ0bl7O/y1hg6Vhg2TKiqkefM6/OUAAAAAn0Bx8gExA86QJCUUFEseGMujZq/Tc8955OUAAAAAr0dx8gFJg0dKkqLLXLI9cPzcdddJkZHSunXSN990+MsBAAAAXo/i5AMyUgdojzOwngo3rurw1+vSRbrmGmeeQSIAAAAAipNPiAiJUG5CmCTpwNqvPfKat9zi3L76qlRc7JGXBAAAALwWxclHHE7pIkk6unmdR15v9Gipb1+ptFR64w2PvCQAAADgtShOPuJYRqokybVtm0dez7JODBLx/PMeeUkAAADAa1GcfITVs7ckKWLXHo+95k03SSEh0ldfSevXe+xlAQAAAK9DcfIRkf0HS5K65B/y2GsmJ0sTJjjzs2Y55zstXSpVV3ssAgAAAOAVKE4+otugsyRJiYfKpfJyj73ugAHO7VtvSddfL114oZSZKS1Y4LEIAAAAgHEUJx+R3vdMlYRKQbZUsWOrR15zwQJp9uyGy/PypMmTKU8AAAAIHBQnH5HYOUk5cZYkaf+a5R3+etXV0p13Srbd8LGaZTNmcNgeAAAAAgPFyUdYlqV9Sc5VcD1xEdzPPpN27276cduWcnOd9QAAAAB/R3HyIcVpiZKkyi2bOvy18vPdux4AAADgyyhOPqQqs4ckKThnV4e/VkqKe9cDAAAAfBnFyYeEZvWXJEXv3tfhrzV6tJSW5lwItzGWJaWnO+sBAAAA/o7i5ENiBwyVJCXuLWl81AY3Cg6WnnzSmW+qPD3xhLMeAAAA4O8oTj4kafDZckmKrHDJ3ru3w19v0iTn+k3du9dfHhHhLJ80qcMjAAAAAF6B4uRDMhOzlBvrzBdu+M4jrzlpkpSTIy1ZIv3xj84y25Yuu8wjLw8AAAB4BYqTD4kIiVBefJgk6eC6FR573eBgacwYaeZMqVcvqbxc+ugjj708AAAAYBzFycccSo2TJB3dvNbjr21Z0pVXOvNvv+3xlwcAAACMoTj5mLIeqZIke/sOI68/caJz++67UlWVkQgAAACAx1GcfIzVu7ckKXJXnpHXP+ccKT5eOnRI+vxzIxEAAAAAj6M4+ZiofoMlSV3zjxh5/ZAQacIEZ57D9QAAABAoKE4+Jv60s53bIxXS0aNGMtQ9z6mDLycFAAAAeAWKk4/pkXm6joQ785XbthjJcOmlUmSktHOn9P33RiIAAAAAHkVx8jGJnZOU3c352Pav/dpIhqioE9dxeucdIxEAAAAAj6I4+RjLsrQ/qbMkqWjDKmM5akbX4zwnAAAABAKKkw8qTk+UJFVs22wswxVXSEFB0urVUk6OsRgAAACAR1CcfFB1ZoYkKTRnp7EM3bpJo0c78xyuBwAAAH9HcfJBYX37S5Kid+83moPD9QAAABAoKE4+KHbgMElSwr5SyeUylqNmWPLPPpMOHjQWAwAAAOhwFCcflDJwhCqDpPAqW3ZenrEcPXtKp58uVVdL771nLAYAAADQ4ShOPigzvo92xjrzhRu+M5ql7sVwAQAAAH9FcfJBESER2pPgXAX30PqVRrPUnOf0739Lx44ZjQIAAAB0GIqTjzrcPU6SdGzzeqM5zjhD6tFDOnpU+vhjo1EAAACADkNx8lFlPbpLkuzt243msCxG1wMAAID/ozj5qKDefSRJkbn5hpOcOM9p0SJnoAgAAADA31CcfFSnfqdJkrrlHzEbRM6FcLt2lQ4ckL780nQaAAAAwP0oTj4qfvAISVKX4kqpqMholtBQ6YornHkO1wMAAIA/ojj5qIwep2l/lDNfuXWz2TA6cZ7TO+9Itm00CgAAAOB2FCcfldgpUTlxzse3f93XhtNIl10mRURI27dL680O9AcAAAC4HcXJR1mWpf3J0ZKk4g2rzYaR1KmTdMklzjyH6wEAAMDfUJx8WElakiSpcpv5Q/UkhiUHAACA/6I4+bDqXpmSpNCduWaDHDdhgnNdp2+/lXK9IxIAAADgFhQnHxbet78kKWb3AcNJHImJ0rnnOvOLFpnNAgAAALgTxcmHdRl0piQpfn+pVFVlOI2j5mK4HK4HAAAAf0Jx8mGp/YarPFgKdUn2rl2m40g6UZyWLpUOHzYaBQAAAHAbipMPy4zrpewuznzRxtUmo9Tq21caNMjZAfb++6bTAAAAAO5BcfJhESERykuMkCQdWrfCcJoT6l4MFwAAAPAHFCcfV5gaJ0k6tsV7rjpbU5w++EAqKzMaBQAAAHALo8Vp9uzZOuussxQdHa3ExERNnDhRmzfXvyaRbduaNWuWUlNTFRkZqTFjxmj9eu8pCaaVZXSXJFnbdxhOcsKZZ0rdu0slJdInn5hOAwAAALSf0eK0bNkyTZ8+XcuXL9fixYtVVVWlsWPHqrS0tHadRx55RHPmzNFTTz2lFStWKDk5WZdeeqmKi4sNJvcewb37SpIicwsMJznBshhdDwAAAP7Fsm3bNh2ixv79+5WYmKhly5bp/PPPl23bSk1N1YwZM/Sb3/xGklReXq6kpCT96U9/0i9+8YsWt1lUVKTY2FgVFhYqJiamo38Ej3t3wZ90xdX3qDgqRNGllabj1Fq8WBo7VkpKkvbskYI4KBQAAACGtacbeNWvs4WFhZKkuDjnvJ3s7GwVFBRo7NixteuEh4frggsu0JdfftnoNsrLy1VUVFRv8mcJg0dIkqKPVkmHDhlOc8IFF0ixsdLevdLXX5tOAwAAALSP1xQn27Z111136bzzztPgwYMlSQUFzuFnSUlJ9dZNSkqqfexks2fPVmxsbO2Unp7escEN69l9kPZ0duYrt25ufmUPCguTLr/cmedwPQAAAPg6rylOt99+u9asWaNXX321wWOWZdW7b9t2g2U17r33XhUWFtZOubm5HZLXWyREJSinm/MxHljrXbt2akbXW7hQ8p4DQgEAAIC284ridMcdd2jRokVasmSJ0tLSapcnJydLUoO9S/v27WuwF6pGeHi4YmJi6k3+zLIsHUxyfsbiTd8bTlPfuHHOnqetW6VNm0ynAQAAAE6d0eJk27Zuv/12LViwQJ988ol69uxZ7/GePXsqOTlZixcvrl1WUVGhZcuW6ZxzzvF0XK9Vku6UyKqtWwwnqS8mRrr4Ymeei+ECAADAlxktTtOnT9e8efP0yiuvKDo6WgUFBSooKNCxY8ckOXtTZsyYoYcfflgLFy7UunXrNG3aNEVFRen66683Gd2ruHpmSpLCdnrfYYkMSw4AAAB/YLQ4zZ07V4WFhRozZoxSUlJqp9dff712nZkzZ2rGjBm67bbbNHz4cOXl5emjjz5SdHS0weTeJbzfQElSbN4Bw0ka+uEPnduvv3aGJQcAAAB8kVddx6kj+Pt1nCTp4y/n6ZJzb5TLkoLKyp0Ti7zIqFHS8uXS3LnSrbeaTgMAAIBA5TfXccKpSe87XKWhUpAt2Tk5puM0UDO6Huc5AQAAwFdRnPxARtdM7ejqzBdtWGU2TCNqitN//iP5+fWIAQAA4KcoTn4gIiRCexIjJEmH1q80nKahfv2cqbJS+uAD02kAAACAtqM4+YnC1G6SpPItGwwnaVzNXidG1wMAAIAvojj5iYoM58LB1o5sw0kaV1Oc3n9fqqgwGgUAAABoM4qTnwjq01eSFJVbYDhJ40aMkJKTnXOcli41nQYAAABoG4qTn+jcf4gkKb6gUPLCEeaDgk5c04nD9QAAAOBrKE5+ImnQ2XJJiix3Sfv2mY7TqLrDkrtcRqMAAAAAbUJx8hM9k/tr9/FreFVu3Ww2TBMuukjq3Fnas0da6X2D/wEAAABNojj5iYSoBOV0cz7OA+u+MZymceHh0uWXO/NcDBcAAAC+hOLkJyzL0oHkWElSycbvDadp2pVXOrec5wQAAABfQnHyI0d7JEuSqrdvNZykaZdfLoWESBs2SFu2mE4DAAAAtA7FyY+4evaUJIXl7DacpGldukgXXujMc7geAAAAfAXFyY9E9BsoSYrdc9BwkubVjK7H4XoAAADwFRQnPxI3cLgkqdvhMunYMcNpmlZzPaevvpL27jWbBQAAAGgNipMfSe95ugrDnXl7xw6zYZqRliYNH+5cp/df/zKdBgAAAGgZxcmPZHTN1PauznzRhlVmw7SAw/UAAADgSyhOfiQiJEL5iZGSpMMbvjOcpnk1xenjj6WSEqNRAAAAgBZRnPxMYfdukqTyLRsMJ2newIFSnz5Sebn073+bTgMAAAA0j+LkZyoz0yVJQdk5ZoO0wLK4GC4AAAB8B8XJzwT3zpIkdcr1/uHqag7Xe/ddqbLSaBQAAACgWRQnPxM98AxJUnxBoeRymQ3TglGjpIQE6cgR6dNPTacBAAAAmkZx8jNJA4arypLCqmwpP990nGYFB5+4ptM775jNAgAAADSH4uRneiVkaWcXZ75iy0ajWVqj7rDktm0yCQAAANA0ipOfSYhK0M4452M9tH6l4TQtu/hiKSpKys2VVnn3pacAAAAQwChOfsayLB1I6SJJKtm4xmyYVoiMlMaNc+YZXQ8AAADeiuLkh46lJ0uSXNu3Gk7SOnUP1wMAAAC8EcXJD9m9e0mSwnfmGU7SOj/4gTNQxNq10o4dptMAAAAADVGc/FBk1iBJUpc9Bw0naZ24OOn88515RtcDAACAN6I4+aG4QcMlSbFFFVJxseE0rcPhegAAAPBmFCc/1KPHaToQ6czb27ebDdNKV17p3H7+uXTggNksAAAAwMkoTn4oo0uGtsc580UbVxvN0loZGdLQoZLLJb37ruk0AAAAQH0UJz8UERKhgkRnl9OR9d8ZTtN6NXudOFwPAAAA3obi5KeKusdLkiq2bjScpPVqznP66CPp6FGjUQAAAIB6KE5+qjKzhyQpKDvHbJA2GDJEysyUjh1zyhMAAADgLShOfiq0Tz9JUufd+wwnaT3LYnQ9AAAAeCeKk5+KHniGJKnbvmKpqspsmDaoKU7vvutTsQEAAODnKE5+KjlrmMqDpZBqW9q923ScVjv3XOeCuAcPSl98YToNAAAA4HBbcdqyZYvOPfdcd20O7dQrvq+yuzjzFVs3Gc3SFiEh0oQJzjyH6wEAAMBbuK04VVZWavny5e7aHNopISpBO7sFS5IOr1tpOE3b1D3PybZNJgEAAAAcHKrnpyzL0sGULpKk0s1rzYZpo7FjpchIKSdHWutb0QEAAOCnWl2cbr31Vv3tb3/TypUrVVFR0ZGZ4CbHeqRIklzbthpO0jZRUdKllzrzHK4HAAAAbxDS2hXXrFmjl19+WaWlpQoNDdXAgQM1bNgwnXnmmRo2bJiCgth55W2sXr0krVP4rj2mo7TZxInSokVOcbr/ftNpAAAAEOhaXZy+/PJL2batTZs26bvvvqudFixYoMLCQknO4WHwHpH9BktapK57DpmO0mZXXCEFBUmrVkk7d0oZGaYTAQAAIJC1ujhJTjEaMGCABgwYoClTptQu3759u7799lutXr3a3fnQDt0GDpckdS6tlA4flrp2NZyo9RISpPPOkz79VHrnHelXvzKdCAAAAIHMLcfX9e7dW9dee60efvhhd2wObpLRfaDyOzvz9rZtZsOcgprR9d55x2gMAAAAgFH1/FlGlwztOL6TqXjj92bDnIIrr3Ruly2TDvne0YYAAADwIxQnPxYREqH8pChJ0pGNqwynabtevaTTTpOqq6X33jOdBgAAAIGM4uTnirvHS5Iqtmw0nOTU1L0YLgAAAGAKxcnPVWU6w9EF5+w0nOTU1BSnf/9bOnbMaBQAAAAEMIqTnwvt20+SFJ27z3CSUzN0qJSeLpWWSv/5j+k0AAAACFQUJz8XM+AMSVLXAyVSRYXZMKfAsk4MEsHhegAAADCF4uTnUvsO09EQKdiWcyVZH1RzuN6iRc5AEQAAAICnGS1On376qSZMmKDU1FRZlqW3T9qlMG3aNFmWVW8aOXKkmbA+qldc79ohySu2bjYb5hSdf77UpYu0f7/01Vem0wAAACAQGS1OpaWlOv300/XUU081uc64ceOUn59fO73//vseTOj7EqISlNMtWJJ0eP0Kw2lOTWio9IMfOPNcDBcAAAAmhJh88fHjx2v8+PHNrhMeHq7k5GQPJfI/lmXpcGoXadNBHd28znScUzZxovTyy9LChdIjjzjnPgEAAACe4vXnOC1dulSJiYnKysrSz372M+3b1/zocOXl5SoqKqo3BbpjPbpLkuzt2w0nOXWXXSaFh0vbt0sbNphOAwAAgEDj1cVp/Pjxevnll/XJJ5/oz3/+s1asWKGLLrpI5eXlTT5n9uzZio2NrZ3S09M9mNg7Wb16SZIidu4xnOTURUdLl1zizDO6HgAAADzNq4vTj370I/3gBz/Q4MGDNWHCBH3wwQfasmWL3nvvvSafc++996qwsLB2ys3N9WBi7xTV/zRJUtf8w5JtG05z6mpG1+M8JwAAAHia0XOc2iolJUUZGRnaunVrk+uEh4crPDzcg6m8X/zA4XJJiiyrcoamS0w0HemUTJjgnNu0YoW0e7eUlmY6EQAAAAKFV+9xOtnBgweVm5urlJQU01F8SmZSP+2OceZ9+TynpCRp1ChnftEis1kAAAAQWIwWp5KSEq1evVqrV6+WJGVnZ2v16tXatWuXSkpKdPfdd+urr75STk6Oli5dqgkTJig+Pl5XXXWVydg+J6NLRu21nIo3rjaapb1qDtfjPCcAAAB4ktHitHLlSg0dOlRDhw6VJN11110aOnSo7r//fgUHB2vt2rW68sorlZWVpalTpyorK0tfffWVoqOjTcb2OREhEdqbFCVJKtyw2myYdqopTkuWSEeOmEwCAACAQGL0HKcxY8bIbmawgn//+98eTOPfitMSpW9yVLl1k+ko7dK3rzRggLRxo/TBB9J115lOBAAAgEDgU+c44dRVZ2ZIkkJydhpO0n4crgcAAABPozgFiNCs/pKkznn7DSdpv5ri9P77UjOX9AIAAADchuIUIGIHOOeRxR08Kh07ZjhN+wwfLqWmSiUl0iefmE4DAACAQEBxChDdew5RYc3lrXJyTEZpt6Ag6cornXkuhgsAAABPoDgFiF5xvWuHJK/c4tsDREj1i5PLZTYLAAAA/B/FKUAkRCVoZ7dgSdLhDd8aTtN+F14oxcRIBQXSN9+YTgMAAAB/R3EKEJZl6XCKs8vp6KZ1htO0X1iYdPnlzjyj6wEAAKCjUZwCSHlGd2dmx3azQdyEYckBAADgKRSnAGL17iNJiti1x3AS9xg/XgoNlTZvljb5/mlbAAAA8GIUpwAS1W+wJKlr/hG/GFEhJka66CJnntH1AAAA0JEoTgEkof+ZqrKk8EqXM6qCH+BwPQAAAHgCxSmA9EzM0q5YZ97ets1sGDf54Q+d2+XLpfx8s1kAAADgvyhOASSjS4a2xznzxZu+NxvGTVJTpREjnPl//ctsFgAAAPgvilMAiQiJ0N7ETpKkog2rzYZxIw7XAwAAQEejOAWYkvRESVLVts2Gk7hPTXH6z3+koiKjUQAAAOCnKE4BprpnpiQpJHuX2SBu1L+/lJUlVVRIH35oOg0AAAD8EcUpwIT37S9Jis7bbziJ+1gWh+sBAACgY1GcAkzswGHObWGZVFxsOI37XHmlc/v++86eJwAAAMCdKE4BJr3HYB2MPH4nO9toFnc6+2wpKUkqLJSWLTOdBgAAAP6G4hRgenXtpe1dnfnKrf4zQERw8IlrOnG4HgAAANyN4hRgEqIStKtbsCTpyPpvDadxr5rznN55R7Jto1EAAADgZyhOAcayLB3u3k2SdHTLesNp3Ouii6ROnaS8POlb/+qEAAAAMIziFIDKe6Q6M9u3mw3iZhER0vjxzjyH6wEAAMCdKE4BKKh3X0lSVG6+4STux7DkAAAA6AgUpwDUqf9pkqQuBYVSdbXhNO51+eVSSIi0fr20davpNAAAAPAXFKcAlJg1VBVBUmi1Le3ebTqOW3XtKo0Z48y/847RKAAAAPAjFKcA1Cu+r7KPD0lub9tmNkwHqLkYLsUJAAAA7kJxCkCZXTK143hxKtm0xmyYDlBTnL74Qtq3z2wWAAAA+AeKUwAKDwnX3qTOkqSijd8bTuN+6enSmWc613L6179MpwEAAIA/oDgFqNL0RElS1bZNhpN0DEbXAwAAgDtRnAJUdc9MSVJojn8NDlGj5nC9xYulkhKzWQAAAOD7KE4BKrzvAElSTN5+w0k6xuDBUq9eUnm59NFHptMAAADA11GcAlSXgcMkSZ1LKqTDhw2ncT/L4nA9AAAAuA/FKUBldB+ogk7H7+zYYTRLR6kpTgsXSv/8p7R0qd9d7xcAAAAeQnEKUL269qodkrxy62azYTrI3r1SUJBzjtNNN0kXXihlZkoLFphOBgAAAF9DcQpQCVEJ2hkfIkk6suE7w2ncb8EC6dprJZer/vK8PGnyZMoTAAAA2obiFKAsy1Jhapwk6djm9YbTuFd1tXTnnc51nE5Ws2zGDA7bAwAAQOtRnAJYeWaaJMnKzjacxL0++0za3cwo67Yt5eY66wEAAACtQXEKYMG9+kqSonblG07iXvmt/HFaux4AAABAcQpgnQYMkSR12VckVVYaTuM+KSnuXQ8AAACgOAWw5D5n6GiIFGxL2rnTdBy3GT1aSktzruXUGMuS0tOd9QAAAIDWoDgFsF5xvWuHJLe3bzcbxo2Cg6Unn3TmGytPti098YSzHgAAANAaFKcAltkls7Y4lWz83mwYN5s0SXrrLal798YfT0jwbB4AAAD4NopTAAsPCde+5M6SpOJN/lWcJKc85eRIS5ZIr7zi3N5yi/PY9OlSVZXReAAAAPAhIaYDwKzS9GRJ21S9bYvpKB0iOFgaM+bE/cGDpYULpbVrpaefdq73BAAAALSEPU4Bzu6ZKUkK3dnMhY/8SHy8NHu2M3///VJBgdk8AAAA8A0UpwAXnjVQkhSbd9AZNSEA3HKLdNZZUlGRNHOm6TQAAADwBRSnANd1wFC5JEUeq5QOHDAdxyOCg53D9CxL+uc/pU8/NZ0IAAAA3o7iFOAyk/srL+b4nR07jGbxpLPOkn72M2d++nS/uv4vAAAAOgDFKcD17nriWk6VWzaZDeNhDz8sxcVJ69Y5e6AAAACAplCcAlx8VLx2xTuDKx7ZsMpwGs/q1k364x+d+QcekPLzzeYBAACA96I4BTjLslSY2k2SVL5lveE0nnfLLdKIEQwUAQAAgOZRnKDyzHRJkpWdYzaIAUFBJwaKmDePgSIAAADQOIoTFNK7rySpU25gXtRo+HDp5z935hkoAgAAAI0xWpw+/fRTTZgwQampqbIsS2+//Xa9x23b1qxZs5SamqrIyEiNGTNG69cH3uFkHa1z/yGSpC4HSqSyMsNpzPjDH5xznhgoAgAAAI0xWpxKS0t1+umn66mnnmr08UceeURz5szRU089pRUrVig5OVmXXnqpiouLPZzUv6X2HKKisON3cnJMRjGm7kAR99/PQBEAAACoz2hxGj9+vB566CFNmjSpwWO2beuJJ57Qfffdp0mTJmnw4MF66aWXdPToUb3yyisG0vqv3t361A5Jbm/bZjaMQTff7AwUUVws/frXptMAAADAm3jtOU7Z2dkqKCjQ2LFja5eFh4frggsu0Jdfftnk88rLy1VUVFRvQvMyYjO0Pc6ZL9m0xmwYg+oOFPHyy9KyZaYTAQAAwFt4bXEqKHAGKkhKSqq3PCkpqfaxxsyePVuxsbG1U3p6eofm9AfhIeHanxQtSSrZ9L3hNGYNHy794hfOPANFAAAAoIbXFqcalmXVu2/bdoNldd17770qLCysnXJzczs6ol842iNZklQdwIfq1agZKGL9eqmJ0+8AAAAQYLy2OCUnO7/In7x3ad++fQ32QtUVHh6umJiYehNaZvfsKUkK30nRjIs7MVDEAw9Ie/aYzQMAAADzvLY49ezZU8nJyVq8eHHtsoqKCi1btkznnHOOwWT+KaLfIElS7J5Dkm0bTmMeA0UAAACgLqPFqaSkRKtXr9bq1aslOQNCrF69Wrt27ZJlWZoxY4YefvhhLVy4UOvWrdO0adMUFRWl66+/3mRsvxTX7wxVW1JYRTVjccsZKOKZZ5yBIl55RVq61HQiAAAAmGS0OK1cuVJDhw7V0KFDJUl33XWXhg4dqvvvv1+SNHPmTM2YMUO33Xabhg8frry8PH300UeKjo42Gdsv9Ursp12xx+/s2GE0i7c480zp1lud+dtvZ6AIAACAQGbZtn8fl1VUVKTY2FgVFhZyvlMz9pfu1/enJeqSbKny+b8p9Oafmo7kFQ4dkrKypIMHpT//WbrrLtOJAAAAcKra0w289hwneFZ8VLxy40MkSYUbVxlO4z3i4qQ//cmZZ6AIAACAwEVxgiRn2PfC7vGSpPItGw2n8S4/+Yl09tlSSYl0992m0wAAAMAEihNqVWY6FwsOys4xG8TL1B0o4tVXGSgCAAAgEFGcUCu4d5YkqdPuvYaTeJ9hw04MFDF9OgNFAAAABBqKE2pFDzhdkhRz+KhzXBrqeeghKT5e2rBB+stfTKcBAACAJ1GcUCs94zQdjDx+JzvbaBZvVHegiFmzpLw8o3EAAADgQRQn1OrVtZd2dHXm7W3bzIbxUtOmSSNHMlAEAABAoKE4oVZGbEZtcSrZvNZsGC8VFCQ9/bQzUMRrr0lLlphOBAAAAE+gOKFWeEi49idHS5JKN35vOI33GjZM+uUvnXkGigAAAAgMFCfUcyw9RZLk2s6hes2pGShi40bpySdNpwEAAEBHozihHrt3L0lS+C5GPmhO167SI4848wwUAQAA4P8oTqgnMmuQJCk2/7BUXW04jXebOlUaNUoqLWWgCAAAAH9HcUI9CVlDVREkhVS52I3SgpqBIoKCnIEiPvnEdCIAAAB0FIoT6ukZ30c5XY7f2b7dZBSfMHToiYEibr9dqqgwmwcAAAAdg+KEenp17aXtcc585dbNZsP4iN//XkpIYKAIAAAAf0ZxQj3xUfHK7RYqSSrasMpwGt9Qd6CIBx+Udu82mwcAAADuR3FCPZZlqTgtXpJUvnWT4TS+46abGCgCAADAn1Gc0EBFZrokKTg7x2wQH1J3oIjXX5f+8x/TiQAAAOBOFCc0ENq7nySp8+59hpP4lqFDpdtuc+YZKAIAAMC/UJzQQMzAMyRJnYrLpCNHjGbxNTUDRWzaxEARAAAA/oTihAZ6dB+ovZ2O39mxw2gWX9OlCwNFAAAA+COKExro1bWXtnd15m2u5dRmN90knXOOM1DEf/+36TQAAABwB4oTGsiIzdCO48WpdNMas2F8UN2BIt54Q/r4Y9OJAAAA0F4UJzQQHhKuAykxkqTSzWsNp/FNZ5whTZ/uzDNQBAAAgO+jOKFRx9JTJEn2tm2Gk/iu//1fKTFR2rxZeuIJ02kAAADQHhQnNK5XL0lS+K49hoP4rroDRfzv/0q5uUbjAAAAoB0oTmhUVP/TJEkxe49IlZVmw/iwG2+Uzj2XgSIAAAB8HcUJjUrsPUTHQqRgly3t2mU6js+qO1DEm28yUAQAAICvojihUb269akdWY9rObXP6aczUAQAAICvozihUb269qotTpVbNpkN4wfqDhTx+OOm0wAAAKCtKE5oVHxUvHbFh0qSijd9bziN7+vSRXr0UWeegSIAAAB8D8UJjbIsSyXdEyRJFVs2Gk7jH268UTrvPOnoUemuu0ynAQAAQFtQnNCkyp49JEnBOQwO4Q6W5QwUERwsvfWWtHix6UQAAABoLYoTmhTWt78kKXr3Psm2DafxD0OG1B8oorzcbB4AAAC0DsUJTYrtf4YkKeJohXTwoNkwfuTBB6WkJGnLFgaKAAAA8BUUJzQpI7mfdkcfv7N9u9Es/qTuQBG//z2XyQIAAPAFFCc0qe6Q5DbFya1uuIGBIgAAAHwJxQlNyojN0PY4Z75081qzYfxM3YEi5s+XPvrIdCIAAAA0h+KEJoWHhOtgcowk6egmipO7DRniDBAhSXfcwUARAAAA3ozihGYd65EqSbJ37DCcxD/VHShizhzTaQAAANAUihOaFdS7jyQpYmee4ST+KTZWeuwxZ56BIgAAALwXxQnNiup/miQp+kCRVFZmOI1/mjJFGj1aOnaMgSIAAAC8FcUJzUrOHKziMCnIlpSTYzqOX7Is6amnTgwU8e9/m04EAACAk1Gc0Kxecb21/fiQ5OI8pw4zZIgzQITEQBEAAADeiOKEZtW9llPllk1mw/i5WbOk5GRp61bpz382nQYAAAB1UZzQrPioeO2OD5UkFW9aYziNf4uNlR591Jl/6CEGigAAAPAmFCc0y7IsFacmSJKCP/9CWrpUqq42G8qP1R0oYsYM5+1+9VXedgAAANMs27Zt0yE6UlFRkWJjY1VYWKiYmBjTcXzPggUqnHadYosrTixLS5OefFKaNMlcLj+2dq10xhmSy1V/OW87AABA+7SnG7DHCU1bsECaPFkxdUuTJOXlSZMnO4/D7bZubViaJN52AAAAk9jjhMZVV0uZmdLu3Y0/blnOLpDsbGccbbgFbzsAAEDHYY8T3O+zz5r+7V2SbFvKzXXWg9vwtgMAAHgnihMal5/v3vXQKrztAAAA3onihEZVJyW6dT20TkqKe9cDAACAe3h1cZo1a5Ysy6o3JScnm44VED7LkHJjpEbGKJAk2ZJ2xTjrwX1Gj3bOYbKsptcJCZG6dfNcJgAAAHh5cZKkQYMGKT8/v3Zau3at6UgBIf/oPt05zpk/uTzZkixJ7/V11oP7BAc7Q45LTZenqipp1Cjprbc8lwsAACDQeX1xCgkJUXJycu2UkJBgOlJASIlO0cKB0uRrpbyTBhw5Eu7c3rxaysou8ng2fzdpklOKunevvzw9XXrhBenCC6XSUumaa6R77uHCuAAAAJ7g1cORz5o1S48++qhiY2MVHh6us88+Ww8//LB69erV5HPKy8tVXl5ee7+oqEjp6ekMR95G1a5qZT6ZqbyiPFkuW6N3SiklUn5n6bMe0ltvSldtkuyMDFnffsuxYx2gutoZPS8/3zmnafRoZ49UVZV0773SY485611yifTqq1J8vNm8AAAA3q49w5F7dXH64IMPdPToUWVlZWnv3r166KGHtGnTJq1fv17dmvhFfdasWXrwwQcbLKc4td2CjQs0+Y3JkiRb9f+ZxJRJ2/7RRQl7jkiXXSa99x4XFvKw11+Xbr5ZOnpUyshwLow7bJjpVAAAAN7Lb4vTyUpLS9W7d2/NnDlTd911V6PrsMfJvRZsXKA7P7xTu4tOXFwoOixaxRXFOvtgpL58zlbQsTLpgQekWbPMBQ1Q69ZJV10lbdsmRURIf/2rNHWq6VQAAADeKWCKkyRdeuml6tOnj+bOnduq9dvz5sBR7arWZ7s+U35xvlKiUzQqbZTGvTxOS3OW6jc7UvXHf+xxRjJ47z1p/HjTcQPOkSPSjTdK777r3J8+XZozRwoLMxoLAADA67SnG3j94BB1lZeXa+PGjUrhIjYeFRwUrDGZY3TdaddpTOYYhYeE67WrX1NK5xT9qdceLb6sr2Tb0pQpUk6O6bgBp0sX6Z13Tuzwe/ppZwCJPXtMpgIAAPAvXl2c7r77bi1btkzZ2dn6+uuvNXnyZBUVFWkqxyIZl9Q5SW9c84aCrWBdcdZW7R2UIR0+LE2eLJWVmY4XcIKCnKMl//UvKTZW+vJL6cwzpc8/N50MAADAP3h1cdq9e7euu+469evXT5MmTVJYWJiWL1+ujAyuuuoNzutxnh699FFVhEjnXJanyq6x0rffSr/6leloAeuKK6SVK6XBg6WCAmfP09NPOzsEAQAAcOp87hyntuIcp45l27aufetavbXhLV2fH695zx6UZdvOBYd+8hPT8QJWSYn00586I+9JzoARc+dKkZFmcwEAAJgUMOc4wftYlqXnf/i8+nXrp1dSDuilScevsXXbbdKqVWbDBbDOnZ1rOz32mHMY30svSeeeyyloAAAAp4rihHaLCY/R/GvnKyo0SjcP2q7NI/s65zldfbVz3hOMsCzpv/9bWrzYuTjuqlXS8OHOfQAAALQNxQluMShxkJ694lnZQdLIC7bqaFqSlJ0t3XST5HKZjhfQLrrIOfVs+HDp4EFp3DjpT3/ivCcAAIC2oDjBbaYMmaLpZ03XkUjp8quOyg4Pcy4uNHu26WgBr0cP6bPPpJtvdnrsPfdI11wjFRebTgYAAOAbKE5wqz+P/bPO7n62lnUr1u9/dPx6W//zPxwf5gUiIqTnnpP+7/+k0FBp/nzp7LOlzZtNJwMAAPB+FCe4VXhIuN645g11i+ymB3rt1OeX9nOOCbv+eik313S8gGdZ0s9/Ln36qZSaKm3cKJ11lvT226aTAQAAeDeKE9yuR2wPvXL1K7Jk6ZIRm3VwQIZ04IBzcdzyctPxIGnkSOm776Tzz3cO17vqKul3v5Oqq00nAwAA8E4UJ3SIsb3H6sExD6o8VDp3fL6qusRI33wj3XWX6Wg4LilJ+vhjacYM5/4f/uBcQPfQIaOxAAAAvBLFCR3mvvPv0/g+47U5pkK/uDbKWfjMM9K8eWaDoVZoqPT4485HEhkpffihc+je99+bTgYAAOBdKE7oMEFWkOZNmqeM2Ay9kFqgNyb1cx74+c+ltWvNhkM9U6ZIX34p9ewp7dghjRolvfKK6VQAAADeg+KEDhUXGae3rn1LYcFhum7wZmWPyJKOHXMujltYaDoe6jjjDGnlSumyy5yPaMoU5zC+ykrTyQAAAMyjOKHDDU8drv83/v/JFSSdPWarylKTpK1bpZ/8hKuwepm4OOm996T77nPuP/mkdMkl0t69ZnMBAACYRnGCR/xs2M809fSp2h9la+LkStlhYdLChdJjj5mOhpMEB0sPPeR8PNHRztDlw4ZJy5ebTgYAAGAOxQkeYVmWnvnBMxqSNET/jjukP/8o3XngnnukpUuNZkPjJk6UVqyQBgyQ9uxxhi7/v/9jJyEAAAhMFCd4TFRolN665i3FhMfo1722a+UlAyWXS/rRj5zfzOF1+vWTvv7aOSWtslK69VbpZz+TyspMJwMAAPAsihM8qm+3vnpp4kuSJZ0/YoOOZGVI+/ZJ117LKAReKjpaevNN6Y9/lIKCpOefd/Y+5eaaTgYAAOA5FCd43MT+EzXznJk6FiaNmXBA1dGdpS++kGbONB0NTbAs6Te/ca7zFBfnHMJ35pnSkiWmkwEAAHgGxQlG/OHiP+iCjAv0fXSpZlzX1Vn4xBPSG28YzYXmXXqp9O230tCh0v79zv05czjvCQAA+D+KE4wICQrRa5NfU0rnFD2VmqtFVw10Hrj5ZmnjRrPh0KzMTGcH4Y03StXV0n//t3TddVJpqelkAAAAHYfiBGOSOyfr9cmvK9gK1qTBG5R3Zpbz2/ekSVJxsel4aEZkpPTSS9JTT0khIdLrr0sjR0rbtjmPV1c7gyW++qpzW11tMi0AAED7UZxg1OiM0Xrk0kdUHSyNuHiHKpISpE2bpJ/+lOO/vJxlSdOnO8UoOVlat04aPty5eG5mpnThhdL11zu3mZnSggWGAwMAALQDxQnG/dfI/9LVA67Wnqgq/ehaS3ZIiHOu05NPmo6GVjj3XOe8p3POkQoLpYcflnbvrr9OXp40eTLlCQAA+C6KE4yzLEsvXPmCsrpl6e1u+/TMdX2cB379a+nzz82GQ6ukpkoffyx16tT44zU7D2fM4LA9AADgmyhO8Aox4TGaf+18RYVG6fZem7T2osFSVZVzfaeCAtPx0Apff938ABG27Vz76bPPPJcJAADAXShO8BqDEwfr2SuelSzpnLPXqbhPDyk/X/rxj50SBa+Wn9+69bKzOzYHAABAR6A4watMGTJFtw2/TSXh0qVXFsrVuZO0bJn029+ajoYWpKS0br3bbpNuuUX69FPG/wAAAL6D4gSvM+eyORrRfYS+ji7UvdcnOQsffZSRBbzc6NFSWpoz2l5TgoOlsjLphRekCy6Q+vSRHnyQvVAAAMD7UZzgdcJDwvXmNW+qW2Q3PZK6Q4snDnEemDZN2rLFaDY0LTj4xECIJ5cny3Km11939jTdcosUHS3t2CHNmiX16iWNGSO9+CKX8AIAAN6J4gSv1CO2h16e9LIsWbr8tDXaOyzL+Y366qubH4EARk2aJL31ltS9e/3laWnO8quvdvZMPfecM+bHvHnSJZc4pWrZMunmm51rQt10k/TJJ5LLZebnAAAAOJll2/59lkFRUZFiY2NVWFiomJgY03HQRv+77H/1wNIH1PNYhDa/2Fmh+w5IU6ZI//xn88eEwajqamf0vPx859yn0aOdPVJNyc11PtKXXqq/U7FHD6dETZ3qHNYHAADQHu3pBhQneDWX7dIPXvmBPtz2oa471F0vP10gq7paevppZ5QB+BXblpYvdwrUa685F9Stce65ToG69lopNtZcRgAA4LsoTs2gOPm+g0cPatizw7SrcJde2DZIP5m3XgoNdU6WGTnSdDx0kGPHpEWLpL//XfrooxOH7UVESFdd5ZzydvHFze/JAgAAqIvi1AyKk39YkbdC5714niqqKrR52WnKWrrWOXHmu++khATT8dDB9uxxzod66SVpw4YTy7t3l2680dkT1b+/uXwAAMA3tKcbMDgEfMJZ3c/SX8b9RbKks0et19Ge6dLu3dL11zsn1MCvpaZKM2dK69ZJK1ZI06dLcXFSXp70xz9KAwY4Ox/nzpUOHzadFgAA+COKE3zGz8/8uW4ccqOOhLs0ftJRuaIipY8/lh54wHQ0eIhlScOHS0895eyFeustacIE53C9r792TntLTnbOg3rvPamqynRiAADgLzhUDz7laOVRjXxupNbuW6sH9/TT/c9udh5YtMj5DRoBae9e6ZVXnPOh1qw5sTw52RmEcdo0afBgU+kAAIC34BynZlCc/M/Wg1s1/G/DVVRepM++G6rzFq1yhln77jvnSqoIaKtXOwXq5ZelAwdOLB82zClQ110nxccbCgcAAIyiODWD4uSfFm5cqElvTFJolZT/ryx1+36LdMYZ0pdfSpGRpuPBC1RUSB984Awo8e67UmWlszw0VLriCmdAicsvd+7X1dZrUAEAAN/B4BAIOFcNuEp3j7pblSHSuePyVBUf5+xquO0252JACHhhYdKVV0oLFjjnQ/3lL85ep8pKaeFCaeJEZ1S+GTOcfzqSs25mpnThhc64Ixde6NxfsMDYjwEAALwEe5zgs6pcVbr4Hxfr052f6pZDmfrbU7tkuVzSs89KP/uZ6XjwUmvXOnuh5s1zzo2qkZEh7dzZcH3Lcm7fekuaNMkzGQEAQMfgUL1mUJz8W35xvoY9O0wFJQV6fesZuvbl1c6uhi++cIZfA5pQVeVcWPfvf5fefvvEoXyNsSznsmHZ2Ry2BwCAL+NQPQSslOgUvT75dQVbwfpx79XKPn+Ic3LL5MnSwYOm48GLhYQ45zi98YazN6k5ti3l5kq//KUzzPmuXRwRCgBAoKE4weedn3G+/nTJn2QHSWeft1FlGWnOMVc33ODsRli6VHr1VeeWi+WiEaWlrVvvb39zBpbIyJC6dnUGjrjtNufCu59/Lh050qExAQCAQRyqB79g27YmvzlZCzYu0GUlyfrgqSOyysqkmBipqOjEimlp0pNPcrIK6lm61BkIoiUXXijt2ydt3tz0xXXT06XTTqs/9e/vHEEKAADM4hynZlCcAkdhWaHO+ttZ2npoq97+Il1XLs5tuBJn+qMR1dXO6Hl5eY0fgnfyOU4VFdKmTc5AE3Wn3Eb+yUnOYYH9+zcsVD16nPgnCQAAOh7FqRkUp8Cydu9ajXp2hDY+Vqa0IqnR30k50x+NWLDAOTVOql+e2tK1jxyR1q1zStSaNScKVd2dnnXFxEiDBzcsVF27ntrPwDWoAABoHsWpGRSnwLP4+ft06U8fbnnFf/1L+sEP+JM/ai1YIN15p7R794ll6enSE0+c+g7KmoElTt47tWlT0yP5paU1frhfeHjbsnNkKgAA9VGcmkFxCkCvvupcvbQ1Ond2frtMT68/1V0WHd2xeeFVPLXXpqLCOVfq5EK1a1fj64eESFlZJ4rUkCHObUaGc0HfyZMbHmbIkakAANRHcWoGxSkAtfZM/9aKjW1Ypk6+HxXlvterwXFXAamw8MThfnWnpkbs69zZKWEVFY0/zpGpAACcQHFqBsUp8FRXVmhvQpSSC6sbHW/fJSm/S7CScw4oeO8+5ziq3FznGKea+ZqpsLB1LxoX13y5SkuTIiJa/0P4+nFXlD63sm1n4Iq6502tXStt3Nj8hXvrGj1aGjhQSkyUEhIa3nbr5uzVMoV/MgAAT6A4NYPiFHiW5izVX35zod56w7lftzy5jt9OvlZae24f9U/or+7R3ZUanaru0d3VPebEfFxknKySkoaF6uT7JSWtC5aQ0Hi5qlnWvbszZnXNKAW+etwVpc9jKiud869mzmz/tizL6f8nF6rGSlZiorOuu94WX/8nAwDwHRSnZlCcAs+ra1/V9Quu11UbpCc/lNLrjGi2K0aaMU5aOLDl7YQHhzslKqa7U6pqClbMifnUzimKPFrR+N6qusuOHWv5BS3L+Y300KHmdyMkJEivvSZFRjpFq+4UGtpwWXCw5wbAOF76bNuuN6KhbVnOfUqf27X2yNQ775S6dJH273euRVX39uDBxodhb05QkLOXqqlidfJt167Oc07m638nkHyqawNAwKM4NYPiFHiW5izVhS85v0kGuaTRO6WUEim/s/RZhuQ6/svbwxc9rG5R3ZRXlKe84jztKd6jvOI85RXl6eCxg61+vbjIuBN7rE7aa9U9prtSO6cosTxEQbvzmj4kcPfupk9SaS/LalimWpoaK2AtTSEh0j33yD50qNFh4G1JVlKS9OGHzmGLoaFNTyEhnh/t0EdLX91rUFl2hUYnPaOU0O3Kr+ytz/beJtsKa/Ecp+pqpzw1Vqoauz10qO05g4OdolW3THXrJs2b5wzXHqTGs3fvLuXkeG8RWbBAmnFnhXpVnsi+I/Q2PfFkmDf+c2mguqJaa5/5TEe35yuqd4pOu220gsO89M0+iU9np2wDxlCcmkFxCjzVrmplPpmpvKI82Wr4z9uSpbSYNGXfma3goMb/T1VWVab84vwThaqRcpVXnKeyqrJWZQoJClFK55QGe6xqylX3zqnqXhamzv94Vfrd71reYGqqs8epZlSAulNrT3rxdiEhzZcrd07BwdLs2bKPHGm+9H36qXPxpU6dnAFBvOQ3nQULpHm/mqknC+covaS6dnlu52DdGXuXbvjLI279Jb6qSjpwoPmCVXf+8OHmt3dV96azL8x7RKGhziAYNW97VFT9+ZPvt3U+MrLxvWEt8fT77m7LZy5Qjzl3KrX6xB7WPcFp2nXXkxr5iBcHl29n9/WyXVFWoUWPPaOSnO3qnNlbP7z7NoVFhJmO1SpkN8Pbsvt9cXrmmWf06KOPKj8/X4MGDdITTzyh0aNHt+q5FKfAtGDjAk1+w7maad3ydHzfgd669i1NGtC+/0PZtq0jZUdqi1TdUrWn5ETZ2luyt9EC15hxuRH64PmWy9hHf7tXh0eertDgUIUGhSokKOTEvBWsMJelsGoptMpWaLWtsGoppMql0GpbIdW2QipdCql2KaTKpeDKagVXVTdewpoqZo0st3fskLVqVcvvW0yMrOBgZzs1k8vV4vO8Uni48xt4zVTzG7k7lkVGtnqv2/K/zNSIOx+V1Pg5fd88+WuN/NUj7v3Z26Cy0ilaJ5erJUukoJUz9VZe09knd/+1FuZ1fPbIyLaXrfUvztTLO5rOPjXr15rxyiMt7mCt6e6etHzmAo149Ooms3/z6/leW0B8Obuvl+1/3D5TF77UMPuSqXfppqfM/TemNchuhjdm9+vi9Prrr+vGG2/UM888o3PPPVf/93//p+eee04bNmxQjx49Wnw+xSlwLdi4QHd+eKd2F534i2R6TLqeGPdEu0tTW1S5qlRQUtCgXJ28B6u4olhBLinnCal7kZocEXB3jNRzxolDDt3BkqXQ4OMFLCj0lOYzV+Xo/z26rsXXmnX/BSo99ywFWUG1U7BtOYXOZSu0Ws58I1Ooy3ksuMrl3Fa7au8HVx+fquzjty4FV1c7t1UuBdUsq6pWUFX18fvOfGT2bnVd2XJ2V1iorMoqWZ76z2YrCpYrPFxHX/w/dSq3G91b5pJUGGUp9ukXFBQa6uxasazGp6YeO5XntGJ7X39TqYz/PkeJR11N/nvfGxWk7x74VgMGhqmszDld8FiZpbIy6ehROcvKLGf5sZr70rFjVu39o0cbPufoMUtl5Sdeq7F37+Rlde8HqUKfRg1ScjPZ8zsFaUTpDlUrXC4FyZZVe1t33tmCpZBQS8GhQfVua+ZDw6zaI2jbu4M1xKrWlFlJSq082GT2vLBu+vofexUWGaygIKfYNXXb3GPteW7NP5e6qiuqVRCdpJSKprPvCeumlOK9XnfYXnW1dPPgmXpxU9Nl+yf9f60X1j3iLTuz6/nH7TN1w9NNZ583/dde+0s82c3w1ux+XZzOPvtsDRs2THPnzq1dNmDAAE2cOFGzZ89u8fkUp8BW7arWZ7s+U35xvlKiUzS6x+gmD88zrbi8WM9++6y+ePLuFkcE3DpmsLpFdlOVq0qVrkpVVlfWm690Hb/fyLzL7pi9O6ZKnztckC0tfanl9cZMlZZlShFVUqcKqVOlFFV5Yr69y6KqnGVAY1xyypttSS5ZsiW5LOukZdbxZar3+IlllkLsaiVUtrxne3dYJx0LDqvdX24f34Zqchwvkk09fmK91j/eYJms2vMMbUkxVWUafHRfi9m/jUnVkfDOx9+DIMlS7fugmvfDkmwryHlfLOd1XHLamm1JtoKOr2PV5qg71WxHlo7fWg3WlSzZQc72KiqkG7Z/rJgKNXlIcGGY9FK/8QqPCKq3kiXVa5H1nm+dvD3n3smls/H7Vv3tWfW3X/scl62JK/7VfPZwadHZP6w97tU+/n+C2m00eGL94DWf/4ksDf9wUX9bJz1ek7128fH/0biqNWHpG+pS3nT2I+GWFl10razgRq7J0Mj/r+yT38zazCffb+qogQYfWqPhbJdLV77/sro08YcxW9LhCEuLfnCDrJrfber+O2nkfWqYqf77bp/078xq5fNP/kdkV1fpB28/p65lTf9Rb3d0sJL3HfX4YXt+W5wqKioUFRWlN998U1dddVXt8jvvvFOrV6/WsmXLGjynvLxc5eUn/oxYVFSk9PR0ihN8Qs3AFi2NCLhk6hKNyRxzSq/hsl21Raqp4tWa+ZML2Yb9G5TzwuMtlr7wa36stJg0uWxXy5NasU6dqdpV3ab1a6bC0kP6/Pd5LZa+038TrdDQiFZvt7WHaNZluY4XqpoyVVm/bJ28bESeNGlTy9tdkyjt7SwF2ZJlH/8fYp3boFYus9S2bTS3fqcKqWt5Y2nrKwqVKo7/PtPgf9mNvMUtreOObYS4pIhqtcilxv9NAQCkt37/uCb/boZHX7M9xcng5Q5bduDAAVVXVyspKane8qSkJBUUFDT6nNmzZ+vBBx/0RDzA7Ub3GK20mDS9PTBP7/S3G4wIaAdZSo9J0+gerTvHrzFBVpDCgsMUFuzev/BUu6qVueFNXaPdeuKk0rc7RvqvcdLKkenKnjTP6/b6Lc1ZqjtXONf+OvkX3ZrSN2OctPD6RW0qrLZty5Z9SmWutVPuO/+Ubp3TYpYtD0xXwuXXnMh2vNS5jv/trOr4/Zq/pdUtfScvc9c6hR++oxtnvNBi9vl/uklRl15e+9ya9/Xk25rXaGoddz7fWvqp7v7duy1mf+T341V1/jmyXS5nzHXblmXbsl0up0Q2sky2s67lsmufI5fLKXPVx29PWuaEc8ly2fXWs48vq9meJals2Xe69/WWD0199OoBChs56Pi2nZ/fKZB27TLbtmvLsC37xLjyx5fXfW7tz3J8Oy7Zx3ej2c7i47e2y1btpmoeczmvFbp+u+5cnNNi9rkXpOto78zjP7MtHd+mdfy9li0F1b4vzs9U8/6ceN+loOO31vH32qq9b9ceshvkOv4zHV9mnfQ6NfeT8vfqoh2FLWb/NCNGexPjGyy3m56pv04jfwho8Y84dr2bBs9NPXBIY3YVN78NSUvTOyu/W1ydfYYO66Ttn3y4c8M/XrT0uE56vOn1U44c0Tl7jja/AUlfpUSpoEtsq7ZZk7CxPSmN/yGmFT9PI8uSi4s0fG/LlzP5NjFSe6Oj671ey3+2a3xP0Mksu6VtHf/3f9LSpJJSDd3X8l/GSnK2tyKFF7G9WF5eni3J/vLLL+stf+ihh+x+/fo1+pyysjK7sLCwdsrNzbUl2YWFhZ6IDLTb/A3zbWuWZVuzLFuzVDvVLJu/Yb7piE2qyR58v+wLpsr+8dXObfD98ursVdVVdtqcNHvStbJ3xdT+emfbkr0zRvaka2Wnz0m3q6qrTEdtoKqi3M6LDbarVT93zVQt2bu7BNtVFeWmozbg69l3xzSfPTfWO7N/vPlje1eMms2+M0b2x5s/Nh21AV/OvnLe441mPnlaOe9x01EbePP3j7cq+5u/f9x01AbIboY3Zy8sLDzlbuDVxam8vNwODg62FyxYUG/5r371K/v8889v1Tba8+YApszfMN9Om5NWrzilz0n32uJRl69m99XSZ9u2/dWTv7ar1fCXyZplXz35a9MRm0R2z6uqrrJvur5bs9lvur6bd/6hwJez+3DZLj9Wbu/q3Hz2ndHBdvkxsrsT2TuG3xYn27btESNG2L/85S/rLRswYIB9zz33tOr5FCf4qqrqKntJ9hL7lTWv2Euyl3jlLwJN8dXsvlr6bNv5JT4vNrje/5h2dwn22l/e6yK7583fMN++qok9rFddK6/+N+/L2X21bNu2bb80vfnsL00ne0cgu/u1pxt49eAQ0onhyP/6179q1KhRevbZZ/W3v/1N69evV0ZGRovPZ1Q9AG3hSyMxnqy6skJr5z+jozu3Kyqjt067+jYFh/rGBRLJ7nkLNi7Qf733K/Vcl1d7LmX2aWl6/PInPXrJhlPhy9mX/2Wmetw/R6mFJ0YXyesSrNwH7zJ6vbXWaOyaPLuig7X0Jt+8nhDZO543ZvfbUfVqPPPMM3rkkUeUn5+vwYMH6/HHH9f555/fqudSnAAAaJxP/6HAl7P7aNmWpIqyCi167BmV5GxX58ze+uHdt3l8OOlTRXYzvC273xen9qA4AQAAAJDa1w24vAQAAAAAtIDiBAAAAAAtoDgBAAAAQAsoTgAAAADQAooTAAAAALSA4gQAAAAALaA4AQAAAEALKE4AAAAA0AKKEwAAAAC0gOIEAAAAAC2gOAEAAABACyhOAAAAANACihMAAAAAtCDEdICOZtu2JKmoqMhwEgAAAAAm1XSCmo7QFn5fnIqLiyVJ6enphpMAAAAA8AbFxcWKjY1t03Ms+1Tqlg9xuVzas2ePoqOjZVmW0SxFRUVKT09Xbm6uYmJijGZBx+FzDgx8zoGDzzow8DkHBj7nwNDc52zbtoqLi5WamqqgoLadteT3e5yCgoKUlpZmOkY9MTExfFkDAJ9zYOBzDhx81oGBzzkw8DkHhqY+57buaarB4BAAAAAA0AKKEwAAAAC0gOLkQeHh4XrggQcUHh5uOgo6EJ9zYOBzDhx81oGBzzkw8DkHho76nP1+cAgAAAAAaC/2OAEAAABACyhOAAAAANACihMAAAAAtIDiBAAAAAAtoDi52TPPPKOePXsqIiJCZ555pj777LNm11+2bJnOPPNMRUREqFevXvrrX//qoaRoj7Z8zkuXLpVlWQ2mTZs2eTAx2urTTz/VhAkTlJqaKsuy9Pbbb7f4HL7PvqetnzPfZ980e/ZsnXXWWYqOjlZiYqImTpyozZs3t/g8vtO+5VQ+Z77Tvmfu3LkaMmRI7cVtR40apQ8++KDZ57jru0xxcqPXX39dM2bM0H333adVq1Zp9OjRGj9+vHbt2tXo+tnZ2br88ss1evRorVq1Sr/97W/1q1/9SvPnz/dwcrRFWz/nGps3b1Z+fn7t1LdvXw8lxqkoLS3V6aefrqeeeqpV6/N99k1t/Zxr8H32LcuWLdP06dO1fPlyLV68WFVVVRo7dqxKS0ubfA7fad9zKp9zDb7TviMtLU1//OMftXLlSq1cuVIXXXSRrrzySq1fv77R9d36XbbhNiNGjLBvvfXWesv69+9v33PPPY2uP3PmTLt///71lv3iF7+wR44c2WEZ0X5t/ZyXLFliS7IPHz7sgXToCJLshQsXNrsO32ff15rPme+zf9i3b58tyV62bFmT6/Cd9n2t+Zz5TvuHrl272s8991yjj7nzu8weJzepqKjQt99+q7Fjx9ZbPnbsWH355ZeNPuerr75qsP5ll12mlStXqrKyssOy4tSdyudcY+jQoUpJSdHFF1+sJUuWdGRMGMD3ObDwffZthYWFkqS4uLgm1+E77fta8znX4Dvtm6qrq/Xaa6+ptLRUo0aNanQdd36XKU5ucuDAAVVXVyspKane8qSkJBUUFDT6nIKCgkbXr6qq0oEDBzosK07dqXzOKSkpevbZZzV//nwtWLBA/fr108UXX6xPP/3UE5HhIXyfAwPfZ99n27buuusunXfeeRo8eHCT6/Gd9m2t/Zz5TvumtWvXqnPnzgoPD9ett96qhQsXauDAgY2u687vcsgpJ0ajLMuqd9+27QbLWlq/seXwLm35nPv166d+/frV3h81apRyc3P12GOP6fzzz+/QnPAsvs/+j++z77v99tu1Zs0aff755y2uy3fad7X2c+Y77Zv69eun1atX68iRI5o/f76mTp2qZcuWNVme3PVdZo+Tm8THxys4OLjBXod9+/Y1aLk1kpOTG10/JCRE3bp167CsOHWn8jk3ZuTIkdq6dau748Egvs+Bi++z77jjjju0aNEiLVmyRGlpac2uy3fad7Xlc24M32nvFxYWpj59+mj48OGaPXu2Tj/9dD355JONruvO7zLFyU3CwsJ05plnavHixfWWL168WOecc06jzxk1alSD9T/66CMNHz5coaGhHZYVp+5UPufGrFq1SikpKe6OB4P4Pgcuvs/ez7Zt3X777VqwYIE++eQT9ezZs8Xn8J32PafyOTeG77TvsW1b5eXljT7m1u9ym4eTQJNee+01OzQ01H7++eftDRs22DNmzLA7depk5+Tk2LZt2/fcc49944031q6/Y8cOOyoqyv6v//ove8OGDfbzzz9vh4aG2m+99ZapHwGt0NbP+fHHH7cXLlxob9myxV63bp19zz332JLs+fPnm/oR0ArFxcX2qlWr7FWrVtmS7Dlz5tirVq2yd+7cads232d/0dbPme+zb/rlL39px8bG2kuXLrXz8/Nrp6NHj9auw3fa953K58x32vfce++99qeffmpnZ2fba9assX/729/aQUFB9kcffWTbdsd+lylObvb000/bGRkZdlhYmD1s2LB6Q2BOnTrVvuCCC+qtv3TpUnvo0KF2WFiYnZmZac+dO9fDiXEq2vI5/+lPf7J79+5tR0RE2F27drXPO+88+7333jOQGm1RM0TtydPUqVNt2+b77C/a+jnzffZNjX3GkuwXX3yxdh2+077vVD5nvtO+5+abb679HSwhIcG++OKLa0uTbXfsd9my7eNnRwEAAAAAGsU5TgAAAADQAooTAAAAALSA4gQAAAAALaA4AQAAAEALKE4AAAAA0AKKEwAAAAC0gOIEAAAAAC2gOAEAUIdlWXr77bdNxwAAeBmKEwDAa0ybNk2WZTWYxo0bZzoaACDAhZgOAABAXePGjdOLL75Yb1l4eLihNAAAONjjBADwKuHh4UpOTq43de3aVZJzGN3cuXM1fvx4RUZGqmfPnnrzzTfrPX/t2rW66KKLFBkZqW7duunnP/+5SkpK6q3zwgsvaNCgQQoPD1dKSopuv/32eo8fOHBAV111laKiotS3b18tWrSo9rHDhw9rypQpSkhIUGRkpPr27dug6AEA/A/FCQDgU/7nf/5HV199tb7//nvdcMMNuu6667Rx40ZJ0tGjRzVu3Dh17dpVK1as0JtvvqmPP/64XjGaO3eupk+frp///Odau3atFi1apD59+tR7jQcffFDXXnut1qxZo8svv1xTpkzRoUOHal9/w4YN+uCDD7Rx40bNnTtX8fHxnnsDAABGWLZt26ZDAAAgOec4zZs3TxEREfWW/+Y3v9H//M//yLIs3XrrrZo7d27tYyNHjtSwYcP0zDPP6G9/+5t+85vfKDc3V506dZIkvf/++5owYYL27NmjpKQkde/eXT/5yU/00EMPNZrBsiz97ne/0+9//3tJUmlpqaKjo/X+++9r3Lhx+uEPf6j4+Hi98MILHfQuAAC8Eec4AQC8yoUXXlivGElSXFxc7fyoUaPqPTZq1CitXr1akrRx40adfvrptaVJks4991y5XC5t3rxZlmVpz549uvjii5vNMGTIkNr5Tp06KTo6Wvv27ZMk/fKXv9TVV1+t7777TmPHjtXEiRN1zjnnnNLPCgDwHRQnAIBX6dSpU4ND51piWZYkybbt2vnG1omMjGzV9kJDQxs81+VySZLGjx+vnTt36r333tPHH3+siy++WNOnT9djjz3WpswAAN/COU4AAJ+yfPnyBvf79+8vSRo4cKBWr16t0tLS2se/+OILBQUFKSsrS9HR0crMzNR//vOfdmVISEioPazwiSee0LPPPtuu7QEAvB97nAAAXqW8vFwFBQX1loWEhNQOwPDmm29q+PDhOu+88/Tyyy/rm2++0fPPPy9JmjJlih544AFNnTpVs2bN0v79+3XHHXfoxhtvVFJSkiRp1qxZuvXWW5WYmKjx48eruLhYX3zxhe64445W5bv//vt15plnatCgQSovL9e7776rAQMGuPEdAAB4I4oTAMCrfPjhh0pJSam3rF+/ftq0aZMkZ8S71157TbfddpuSk5P18ssva+DAgZKkqKgo/fvf/9add96ps846S1FRUbr66qs1Z86c2m1NnTpVZWVlevzxx3X33XcrPj5ekydPbnW+sLAw3XvvvcrJyVFkZKRGjx6t1157zQ0/OQDAmzGqHgDAZ1iWpYULF2rixImmowAAAgznOAEAAABACyhOAAAAANACznECAPgMji4HAJjCHicAAAAAaAHFCQAAAABaQHECAAAAgBZQnAAAAACgBRQnAAAAAGgBxQkAAAAAWkBxAgAAAIAWUJwAAAAAoAUUJwAAAABowf8HtkakqUbLuk8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_training_history(history,\n",
    "                      mini_batch_size,\n",
    "                      seq_length,\n",
    "                      len(alphabet_data),\n",
    "                      plot_smooth=True,\n",
    "                      plot_val=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bGCs9Q6ikp6S",
    "outputId": "f9a4abc0-63b4-447b-a214-e972bb949c11"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "with open('illiad_clean.txt','r') as file:\n",
    "  text_data = file.read()\n",
    "\n",
    "text_data = re.sub(r\"\\[.\\]\",\" \",text_data)\n",
    "\n",
    "text_data = re.sub(r\"[^a-zA-Z0-9.,?!:;“”'`’ ]+\", \" \", text_data)\n",
    "\n",
    "text_data = re.sub(r\"[\\s]+\",\" \",text_data)\n",
    "\n",
    "train_split = 0.85\n",
    "n_train = int(len(text_data)*train_split)\n",
    "\n",
    "training_data = text_data[:n_train]\n",
    "validation_data = text_data[n_train:]\n",
    "\n",
    "chars = set(list(training_data))\n",
    "\n",
    "vocab_size = len(chars)\n",
    "\n",
    "char_to_ix = {ch:i for i,ch in enumerate(chars)}\n",
    "ix_to_char = {i:ch for i,ch in enumerate(chars)}\n",
    "\n",
    "print(f'{vocab_size} unique characters')\n",
    "print(f'Training has {len(training_data)} total characters')\n",
    "print(f'Validation has {len(validation_data)} total characters')\n",
    "\n",
    "seq_length = 100\n",
    "\n",
    "print(f'Will train on character sequences of length {seq_length}')\n",
    "print()\n",
    "txt = wrap(training_data[len(training_data)//2:(len(training_data)//2)+200])\n",
    "txt = [line.center(100) for line in txt]\n",
    "txt = '\\n'.join(txt)\n",
    "print('Training data sample: \\n')\n",
    "print(txt)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "XYjwknZjmXll",
    "outputId": "3997af4e-d98f-4551-8435-4993e433721a"
   },
   "outputs": [],
   "source": [
    "mykey = random.PRNGKey(1)\n",
    "\n",
    "seq_length = 100\n",
    "\n",
    "hidden_sizes = [64,64]\n",
    "\n",
    "mini_batch_size = 64\n",
    "\n",
    "history,out_params,out_hidden = train_character_lstm(                  \n",
    "                                  seq_length=seq_length,\n",
    "                                  hidden_sizes=hidden_sizes,\n",
    "                                  mini_batch_size=mini_batch_size,\n",
    "                                  learning_rate=0.02,\n",
    "                                  total_steps=2000,\n",
    "                                  steps_sample_freq=250,\n",
    "                                  initial_key=mykey,\n",
    "                                  training_data=training_data,\n",
    "                                  validation_data=validation_data)\n",
    "\n",
    "plot_training_history(history,\n",
    "                      mini_batch_size,\n",
    "                      seq_length,\n",
    "                      len(training_data),\n",
    "                      plot_smooth=True,\n",
    "                      plot_val=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mE_gChPW5Vpi",
    "outputId": "ef1f8d48-4c1b-4b7e-ca24-cd70efe086e5"
   },
   "outputs": [],
   "source": [
    "key = random.PRNGKey(7)\n",
    "\n",
    "sample_size = 2000\n",
    "\n",
    "key,subkey=random.split(key) #use key to get next key, subkey for random number\n",
    "\n",
    "h_zeros_start = {l:jnp.zeros_like(out_hidden[0][l]) for l in range(2)}\n",
    "c_zeros_start = {l:jnp.zeros_like(out_hidden[1][l]) for l in range(2)}\n",
    "\n",
    "sample_ix = sample(char_to_ix['T'],sample_size,subkey,h_zeros_start,c_zeros_start,out_params[0],temperature=0.6)\n",
    "\n",
    "txt = ''.join(ix_to_char[ix] for ix in sample_ix)\n",
    "txt_wrap = wrap(txt,80)\n",
    "txt_wrap = [line.center(100) for line in txt_wrap]\n",
    "txt = '\\n'.join(txt_wrap)  # \\n aren't in the character set so wrap text to make readable\n",
    "\n",
    "print('----\\n %s \\n----' % (txt,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gQQ6EF0aB3tG"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}