DL_Assignment_1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DL-Assignment-1.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "Y2LuwnJsibi0"
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hpyz5j07rH2E"
      },
      "source": [
        "pip install --upgrade wandb"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CAIu3rObrQiU"
      },
      "source": [
        "!wandb login dd888f73500a67fc53f9191092b22f3946ac0e02"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O6B76YblrTNs"
      },
      "source": [
        "# Init wandb\r\n",
        "import wandb\r\n",
        "\r\n",
        "wandb.init(project=\"assignment-1\", entity=\"ravi-kumar\")\r\n",
        "\r\n",
        "# Declare Hyperparameters\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y2LuwnJsibi0"
      },
      "source": [
        "#Question 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0K-FA-RtiaPs"
      },
      "source": [
        "def q1_plot_samples():\r\n",
        "  (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()\r\n",
        "\r\n",
        "  labels=[\"T-shirt/top\",\"Trouser\",\"Pullover\",\"Dress\",\"Coat\",\r\n",
        "        \"Sandal\",\"Shirt\",\"Sneaker\",\"Bag\",\"Ankle boot\"]\r\n",
        "  p_images, p_labels= [], []\r\n",
        "\r\n",
        "  for i in range(len(x_train)):\r\n",
        "    if len(p_labels) == 10:\r\n",
        "      break\r\n",
        "    if labels[y_train[i]] not in p_labels:\r\n",
        "      p_images.append(x_train[i])\r\n",
        "      p_labels.append(labels[y_train[i]])\r\n",
        "\r\n",
        "  wandb.log({ \"Class Examples\": [ wandb.Image(img, caption=label) for img, label in zip(p_images, p_labels)] })"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W_5xkxQHn3GT"
      },
      "source": [
        "q1_plot_samples()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n6ovTgpBn6Ke"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kk6HBy6TrWM4"
      },
      "source": [
        "from keras.datasets import mnist, fashion_mnist\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "import numpy as np\r\n",
        "from tqdm.notebook import tqdm\r\n",
        "from sklearn.preprocessing import OneHotEncoder\r\n",
        "from sklearn.model_selection import train_test_split\r\n",
        "from sklearn.metrics import mean_squared_error, log_loss, accuracy_score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3vJi8-KR-ZlR"
      },
      "source": [
        "# Model Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7mk75of2rkHa"
      },
      "source": [
        "class FFNNetwork:\r\n",
        "\r\n",
        "    def __init__(self, input_size, output_size=1, hidden_layers=[2], init_func='xavier', act_func='sigmoid', loss_func='ce'):\r\n",
        "\r\n",
        "      # Initializing Network Parameters\r\n",
        "      self.x= input_size\r\n",
        "      self.y= output_size\r\n",
        "      self.h= len(hidden_layers)\r\n",
        "      self.init_func= init_func\r\n",
        "      self.act_func= act_func\r\n",
        "      self.loss_func= loss_func\r\n",
        "      self.sizes= [self.x] + hidden_layers + [self.y]\r\n",
        "      self.W= {}\r\n",
        "      self.B= {}\r\n",
        "      np.random.seed(0)\r\n",
        "\r\n",
        "      # He Initialization\r\n",
        "      if self.init_func=='he':\r\n",
        "        for i in range(self.h+1):\r\n",
        "          self.W[i+1]=np.random.randn(self.sizes[i], self.sizes[i+1]) * np.sqrt(2 / self.sizes[i-1])\r\n",
        "          self.B[i+1]=np.random.randn(1, self.sizes[i+1])\r\n",
        "      \r\n",
        "      # Xavier Initialization\r\n",
        "      elif self.init_func=='xavier':\r\n",
        "        for i in range(self.h+1):\r\n",
        "          self.W[i+1]=np.random.randn(self.sizes[i], self.sizes[i+1]) * np.sqrt(1 / self.sizes[i-1])\r\n",
        "          self.B[i+1]=np.random.randn(1, self.sizes[i+1])\r\n",
        "\r\n",
        "      # Zero Initialization\r\n",
        "      elif self.init_func=='zero':\r\n",
        "        for i in range(self.h+1):\r\n",
        "          self.W[i+1]=np.zeros((self.sizes[i], self.sizes[i+1]))\r\n",
        "          self.B[i+1]=np.zeros((1, self.sizes[i+1]))\r\n",
        "      \r\n",
        "      # Random Initialization\r\n",
        "      else:\r\n",
        "        for i in range(self.h+1):\r\n",
        "          self.W[i+1]=np.random.randn(self.sizes[i], self.sizes[i+1])\r\n",
        "          self.B[i+1]=np.random.randn(1, self.sizes[i+1])\r\n",
        "\r\n",
        "    def perceptron(self, x, w, b):\r\n",
        "      \"\"\" It computes the perceptron output for the inputs passed \"\"\"\r\n",
        "      return np.dot(x, w)+ b\r\n",
        "\r\n",
        "    def activation(self, x):\r\n",
        "      \"\"\" It computes the activation function used \"\"\"\r\n",
        "      # Using tanh function as the activation function\r\n",
        "      if self.act_func == 'tanh':\r\n",
        "        return np.tanh(x)\r\n",
        "      \r\n",
        "      # Using ReLU function as the activation function\r\n",
        "      elif self.act_func == 'relu':\r\n",
        "        return np.maximum(0, x)\r\n",
        "\r\n",
        "      # Using sigmoid function as the activation function\r\n",
        "      else:\r\n",
        "        return 1.0/(1.0 + np.exp(-x))\r\n",
        "\r\n",
        "    def grad_activation(self, x):\r\n",
        "      \"\"\" It computes the gradient of activation function used \"\"\"\r\n",
        "      # Gradient of tanh function\r\n",
        "      if self.act_func == 'tanh':\r\n",
        "        return (1 - np.square(x))\r\n",
        "      \r\n",
        "      # Gradient of ReLU function\r\n",
        "      elif self.act_func == 'relu':\r\n",
        "        return (1.0 * (x>0))\r\n",
        "\r\n",
        "      # Gradient of sigmoid function\r\n",
        "      else:\r\n",
        "        return x * (1-x)\r\n",
        "      \r\n",
        "\r\n",
        "    def softmax(self, y):\r\n",
        "      \"\"\" It computes the softmax of input array passed \"\"\"\r\n",
        "      max=np.max(y)\r\n",
        "      exps=np.exp(y-max)\r\n",
        "      return np.exp(y-max)/np.sum(exps)\r\n",
        "\r\n",
        "    def forward_pass(self, x):\r\n",
        "      \"\"\" It runs a forward pass of Feed Forward Neural Network \"\"\"\r\n",
        "      self.A={}\r\n",
        "      self.H={}\r\n",
        "      self.H[0]= x\r\n",
        "      \r\n",
        "      for i in range(self.h+1):\r\n",
        "        self.A[i+1]=np.matmul(self.H[i], self.W[i+1]) + self.B[i+1]\r\n",
        "        self.H[i+1]=self.activation(self.A[i+1])\r\n",
        "      self.H[self.h+1]=self.softmax(self.A[self.h+1])\r\n",
        "      return self.H[self.h+1]\r\n",
        "\r\n",
        "    def grad(self, x, y):\r\n",
        "      \"\"\" It computes the gradient using backpropgation \"\"\"\r\n",
        "      \r\n",
        "      L=self.h+1\r\n",
        "      self.forward_pass(x)\r\n",
        "\r\n",
        "      self.dW={}\r\n",
        "      self.dB={}\r\n",
        "      self.dA={}\r\n",
        "      self.dH={}\r\n",
        "\r\n",
        "      if self.loss_func=='mse':\r\n",
        "        self.dA[L]=(self.H[L]-y) * self.H[L] * (1-self.H[L])\r\n",
        "      else:\r\n",
        "        self.dA[L]=(self.H[L]-y)\r\n",
        "\r\n",
        "      for k in range(L, 0, -1):\r\n",
        "        self.dW[k]=np.matmul(self.H[k-1].T, self.dA[k])\r\n",
        "        self.dB[k]=self.dA[k]\r\n",
        "        self.dH[k-1]=np.matmul(self.dA[k], self.W[k].T)\r\n",
        "        self.dA[k-1]=np.multiply(self.dH[k-1], self.grad_activation(self.H[k-1]))\r\n",
        "\r\n",
        "    def predict(self, X):\r\n",
        "      \"\"\" It predicts the output for the inputs passed \"\"\"\r\n",
        "      preds=[]\r\n",
        "      for x in X:\r\n",
        "        preds.append(self.forward_pass(x)) \r\n",
        "      return np.array(preds).squeeze()\r\n",
        "\r\n",
        "    def fit(self, inputs, output_labels, epochs=1, lr=0.001, weight_decay=0, display_loss=False, display_accuracy=False, opt_algo='adam', batch_size=128, gamma=0.9, beta=0.9, beta1=0.9, beta2=0.999, epsi=1e-8):\r\n",
        "      \"\"\" It trains the Neural Netword \"\"\"\r\n",
        "      \r\n",
        "      x_train, x_val, y_train, y_val= train_test_split(inputs, output_labels, test_size=0.1, random_state=1)\r\n",
        "\r\n",
        "      accuracy, val_accuracy, loss, val_loss= {}, {}, {}, {}\r\n",
        "      vW, vB, best_W, best_B= {}, {}, {}, {}\r\n",
        "      max_accuracy, max_val_accuracy = 0, 0\r\n",
        "      for i in range(self.h+1):\r\n",
        "        vW[i+1]=np.zeros((self.sizes[i], self.sizes[i+1]))\r\n",
        "        vB[i+1]=np.zeros((1, self.sizes[i+1]))\r\n",
        "\r\n",
        "      m=inputs.shape[2]\r\n",
        "\r\n",
        "      for e in tqdm(range(epochs), total=epochs, unit=\"epoch\"):\r\n",
        "        dW, dB= {}, {}\r\n",
        "\r\n",
        "        for i in range(self.h+1):\r\n",
        "          dW[i+1]=np.zeros((self.sizes[i], self.sizes[i+1]))\r\n",
        "          dB[i+1]=np.zeros((1, self.sizes[i+1]))\r\n",
        "\r\n",
        "        # Gradient Descent/ Batch Gradient Descent/ Vanilla Gradient Descent\r\n",
        "        if opt_algo=='gd' or opt_algo=='bgd' or opt_algo=='vgd': \r\n",
        "\r\n",
        "          for x, y in zip(x_train, y_train):\r\n",
        "            self.grad(x, y)\r\n",
        "            for i in range(self.h+1):\r\n",
        "              if l2_regularisation:\r\n",
        "                dW[i+1]+=self.dW[i+1] + weight_decay * self.W[i+1]\r\n",
        "              else:\r\n",
        "                dW[i+1]+=self.dW[i+1]\r\n",
        "              dB[i+1]+=self.dB[i+1]\r\n",
        "\r\n",
        "          for i in range(self.h+1):\r\n",
        "            self.W[i+1]-= lr* (dW[i+1])/m\r\n",
        "            self.B[i+1]-= lr* (dB[i+1])/m\r\n",
        "\r\n",
        "        # Stochastic Gradient Descent\r\n",
        "        elif opt_algo=='sgd':\r\n",
        "          \r\n",
        "          sample_count= 0\r\n",
        "          for x, y in zip(x_train, y_train):\r\n",
        "            self.grad(x, y)\r\n",
        "            sample_count+=1\r\n",
        "            for i in range(self.h+1): \r\n",
        "              dW[i+1]+= self.dW[i+1] + weight_decay * self.W[i+1]\r\n",
        "              dB[i+1]+= self.dB[i+1]\r\n",
        "\r\n",
        "              if sample_count % batch_size == 0:\r\n",
        "                self.W[i+1]-= lr*dW[i+1]/batch_size\r\n",
        "                self.B[i+1]-= lr*dB[i+1]/batch_size\r\n",
        "\r\n",
        "        # Momentum Based Gradient Descent\r\n",
        "        elif opt_algo=='momentum':\r\n",
        "\r\n",
        "          sample_count=0\r\n",
        "          for x, y in zip(x_train, y_train):\r\n",
        "            self.grad(x, y)\r\n",
        "            for i in range(self.h+1): \r\n",
        "              dW[i+1]+= self.dW[i+1] + weight_decay * self.W[i+1]\r\n",
        "              dB[i+1]+= self.dB[i+1]\r\n",
        "            \r\n",
        "            sample_count+=1\r\n",
        "            if sample_count % batch_size == 0:\r\n",
        "              for i in range(self.h+1):\r\n",
        "                vW[i+1]= (gamma * vW[i+1] + lr*dW[i+1])/batch_size\r\n",
        "                vB[i+1]= (gamma * vB[i+1] + lr*dB[i+1])/batch_size\r\n",
        "                self.W[i+1]-= vW[i+1]\r\n",
        "                self.B[i+1]-= vB[i+1]\r\n",
        "\r\n",
        "        # Nestrov Accelerated Gradient Descent\r\n",
        "        elif opt_algo=='nesterov':\r\n",
        "\r\n",
        "          sample_count=0\r\n",
        "          for x, y in zip(x_train, y_train):\r\n",
        "            self.grad(x, y)\r\n",
        "            sample_count+=1\r\n",
        "            for i in range(self.h+1): \r\n",
        "              dW[i+1]+= self.dW[i+1] + weight_decay * self.W[i+1]\r\n",
        "              dB[i+1]+= self.dB[i+1]\r\n",
        "\r\n",
        "            if sample_count % batch_size == 0:\r\n",
        "              tW, tB= {}, {}\r\n",
        "              for i in range(self.h+1):\r\n",
        "                tW[i+1]= self.W[i+1] - gamma * vW[i+1]\r\n",
        "                tB[i+1]= self.B[i+1] - gamma * vB[i+1]\r\n",
        "                self.W[i+1]= tW[i+1]\r\n",
        "                self.B[i+1]= tB[i+1]\r\n",
        "\r\n",
        "              self.grad(x, y)\r\n",
        "              for i in range(self.h+1):\r\n",
        "                vW[i+1]= gamma * vW[i+1] + lr * self.dW[i+1]\r\n",
        "                vB[i+1]= gamma * vB[i+1] + lr * self.dB[i+1]\r\n",
        "                self.W[i+1]= tW[i+1] - vW[i+1]\r\n",
        "                self.B[i+1]= tB[i+1] - vB[i+1]\r\n",
        "                               \r\n",
        "        # RMSProp Gradient Descent\r\n",
        "        elif opt_algo=='rmsprop':\r\n",
        "\r\n",
        "          sample_count=0\r\n",
        "          for x, y in zip(x_train, y_train):\r\n",
        "            self.grad(x, y)\r\n",
        "            sample_count+=1\r\n",
        "            for i in range(self.h+1): \r\n",
        "              dW[i+1]+= self.dW[i+1] + weight_decay * self.W[i+1]\r\n",
        "              dB[i+1]+= self.dB[i+1]\r\n",
        "\r\n",
        "            if sample_count % batch_size == 0:\r\n",
        "\r\n",
        "              for i in range(self.h+1):\r\n",
        "                vW[i+1]= beta * vW[i+1] + (1-beta) * np.power(dW[i+1], 2)\r\n",
        "                vB[i+1]= beta * vB[i+1] + (1-beta) * np.power(dB[i+1], 2)\r\n",
        "                self.W[i+1]-= (lr/np.sqrt(vW[i+1] + epsi)) * dW[i+1]\r\n",
        "                self.B[i+1]-= (lr/np.sqrt(vB[i+1] + epsi)) * dB[i+1]                \r\n",
        "\r\n",
        "        # Adam Gradient Descent\r\n",
        "        elif opt_algo=='adam':\r\n",
        "\r\n",
        "          sample_count=0\r\n",
        "          for x, y in zip(x_train, y_train):\r\n",
        "            self.grad(x, y)\r\n",
        "            sample_count+=1\r\n",
        "            for i in range(self.h+1): \r\n",
        "              dW[i+1]+= self.dW[i+1] + weight_decay * self.W[i+1]\r\n",
        "              dB[i+1]+= self.dB[i+1]\r\n",
        "\r\n",
        "            if sample_count % batch_size == 0:\r\n",
        "              mW, mB= {}, {}\r\n",
        "              for i in range(self.h+1):\r\n",
        "                mW[i+1]= np.zeros(dW[i+1].shape)\r\n",
        "                mB[i+1]= np.zeros(dB[i+1].shape)\r\n",
        "                mW[i+1]= beta1 * mW[i+1] + (1 - beta1) * dW[i+1]\r\n",
        "                mB[i+1]= beta1 * mB[i+1] + (1 - beta1) * dB[i+1]\r\n",
        "\r\n",
        "                vW[i+1]= beta2 * vW[i+1] + (1 - beta2) * np.power(dW[i+1], 2)\r\n",
        "                vB[i+1]= beta2 * vB[i+1] + (1 - beta2) * np.power(dB[i+1], 2)\r\n",
        "\r\n",
        "                mW[i+1]= (1.0/(1.0- np.power(beta1 , sample_count))) * mW[i+1]\r\n",
        "                mB[i+1]= (1.0/(1.0- np.power(beta1 , sample_count))) * mB[i+1]\r\n",
        "                vW[i+1]= (1.0/(1.0- np.power(beta2 , sample_count))) * vW[i+1] \r\n",
        "                vB[i+1]= (1.0/(1.0- np.power(beta2 , sample_count))) * vB[i+1]\r\n",
        "\r\n",
        "                self.W[i+1]-= (lr/np.sqrt(vW[i+1] + epsi)) * mW[i+1]\r\n",
        "                self.B[i+1]-= (lr/np.sqrt(vB[i+1] + epsi)) * mB[i+1]\r\n",
        "\r\n",
        "        # Nadam Gradient Descent\r\n",
        "        elif opt_algo=='nadam':\r\n",
        "\r\n",
        "          sample_count=0\r\n",
        "          for x, y in zip(x_train, y_train):\r\n",
        "            self.grad(x, y)\r\n",
        "            sample_count+=1\r\n",
        "            for i in range(self.h+1): \r\n",
        "              dW[i+1]+= self.dW[i+1] + weight_decay * self.W[i+1]\r\n",
        "              dB[i+1]+= self.dB[i+1]\r\n",
        "\r\n",
        "            if sample_count % batch_size == 0:\r\n",
        "              mW, mB= {}, {}\r\n",
        "              for i in range(self.h+1):\r\n",
        "                mW[i+1]= np.zeros(dW[i+1].shape)\r\n",
        "                mB[i+1]= np.zeros(dB[i+1].shape)\r\n",
        "                mW[i+1]= beta1 * mW[i+1] + (1-beta1) * dW[i+1]\r\n",
        "                mB[i+1]= beta1 * mB[i+1] + (1-beta1) * dB[i+1]\r\n",
        "\r\n",
        "                vW[i+1]= beta2 * vW[i+1] + (1-beta2) * np.power(dW[i+1], 2)\r\n",
        "                vB[i+1]= beta2 * vB[i+1] + (1-beta2) * np.power(dB[i+1], 2)\r\n",
        "                \r\n",
        "                mW[i+1]= mW[i+1] / (1- np.power(beta1 , sample_count))\r\n",
        "                mB[i+1]= mB[i+1] / (1- np.power(beta1 , sample_count))\r\n",
        "                vW[i+1]= vW[i+1] / (1- np.power(beta2 , sample_count))\r\n",
        "                vB[i+1]= vB[i+1] / (1- np.power(beta2 , sample_count))\r\n",
        "                xW, xB= {}, {}\r\n",
        "                xW[i+1]= beta1 * mW[i+1] + (1-beta1) * dW[i+1] / (1- np.power(beta1, sample_count))\r\n",
        "                xB[i+1]= beta1 * mB[i+1] + (1-beta1) * dB[i+1] / (1- np.power(beta1, sample_count))\r\n",
        "                self.W[i+1]-= ((lr/np.sqrt(vW[i+1] + epsi)) * xW[i+1])\r\n",
        "                self.B[i+1]-= ((lr/np.sqrt(vB[i+1] + epsi)) * xB[i+1])\r\n",
        "\r\n",
        "        # Calculating Loss and Accuracy\r\n",
        "        y_preds=self.predict(x_train)\r\n",
        "        y_val_preds=self.predict(x_val)\r\n",
        "        if self.loss_func=='mse':\r\n",
        "          loss[e]= mean_squared_error(y_train, y_preds)\r\n",
        "          val_loss[e]= mean_squared_error(y_val, y_val_preds)\r\n",
        "        else:\r\n",
        "          loss[e]= log_loss(y_train, y_preds)\r\n",
        "          val_loss[e]= log_loss(y_val, y_val_preds)\r\n",
        "\r\n",
        "        accuracy[e]= accuracy_score(np.argmax(y_preds, axis=1), np.argmax(y_train, axis=1))\r\n",
        "        val_accuracy[e]= accuracy_score(np.argmax(y_val_preds, axis=1), np.argmax(y_val, axis=1))\r\n",
        "\r\n",
        "        if accuracy[e] > max_accuracy:\r\n",
        "          max_accuracy= accuracy[e]\r\n",
        "          max_val_accuracy= val_accuracy[e]\r\n",
        "          best_W=self.W.copy()\r\n",
        "          best_B=self.B.copy()\r\n",
        "        \r\n",
        "        # wandb.log({ 'mnist_accuracy': accuracy[e]})\r\n",
        "        # wandb.log({ 'epoch': e, 'loss': loss[e], 'val_loss': val_loss[e], 'accuracy': accuracy[e], 'val_accuracy': val_accuracy[e]})\r\n",
        "\r\n",
        "      # Plotting Loss\r\n",
        "      if display_accuracy:\r\n",
        "        print('Train Accuracy : ', max_accuracy)\r\n",
        "        print('Validation Accuracy : ', max_val_accuracy)\r\n",
        "\r\n",
        "      # Plotting Loss\r\n",
        "      if display_loss:\r\n",
        "        plt.plot(np.asarray(list(loss.values())))\r\n",
        "        plt.xlabel(\"Epoch\")\r\n",
        "        plt.ylabel(\"Mean Squared Loss\")\r\n",
        "        plt.show()\r\n"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1LH3MPfQb-C_"
      },
      "source": [
        "# Wandb Hyperparameter Tunning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ynRCWrgEg7xK",
        "outputId": "a3909ee1-acd4-43fb-d54e-3f0a205b4058"
      },
      "source": [
        "# Configure the sweep – specify the parameters to search through, the search strategy, the optimization metric et all.\r\n",
        "sweep_config = {\r\n",
        "    'method': 'random', #grid, random\r\n",
        "    'metric': {\r\n",
        "      'name': 'accuracy',\r\n",
        "      'goal': 'maximize'   \r\n",
        "    },\r\n",
        "    'parameters': {\r\n",
        "        'epochs': {\r\n",
        "            'values': [ 5, 10]\r\n",
        "        },\r\n",
        "        'hidden_layers': {\r\n",
        "            'values': [3, 4, 5]\r\n",
        "        },\r\n",
        "        'hidden_layer_size': {\r\n",
        "            'values': [32, 64, 128]\r\n",
        "        },\r\n",
        "        'weight_decay': {\r\n",
        "            'values': [0, 0.0005, 0.5]\r\n",
        "        },\r\n",
        "        'learning_rate': {\r\n",
        "            'values': [1e-3, 1e-4]\r\n",
        "        },\r\n",
        "        'optimizer': {\r\n",
        "            'values': ['sgd', 'momentum', 'nesterov', 'rmsprop', 'adam', 'nadam']\r\n",
        "        },\r\n",
        "        'batch_size': {\r\n",
        "            'values': [16, 32, 64]\r\n",
        "        },\r\n",
        "        'initialisation': {\r\n",
        "            'values': ['random', 'xavier']\r\n",
        "        },        \r\n",
        "        'activation': {\r\n",
        "            'values': ['sigmoid', 'tanh', 'relu']\r\n",
        "        }\r\n",
        "    }\r\n",
        "}\r\n",
        "\r\n",
        "sweep_id = wandb.sweep(sweep_config, entity=\"ravi-kumar\", project=\"assignment-1\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Calling wandb.login() after wandb.init() has no effect.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Create sweep with ID: wf3zsbpf\n",
            "Sweep URL: https://wandb.ai/ravi-kumar/assignment-1/sweeps/wf3zsbpf\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-9_gSf46igdI"
      },
      "source": [
        "def train():\r\n",
        "  config_defaults = {\r\n",
        "        'epochs': 5,\r\n",
        "        'hidden_layers': 3,\r\n",
        "        'hidden_layer_size': 32,\r\n",
        "        'weight_decay': 0,\r\n",
        "        'learning_rate': 1e-3,\r\n",
        "        'optimizer': 'adam',\r\n",
        "        'batch_size': 64,\r\n",
        "        'initialisation': 'xavier',        \r\n",
        "        'activation': 'sigmoid'\r\n",
        "    }\r\n",
        "  wandb.init(config=config_defaults)\r\n",
        "  config= wandb.config\r\n",
        "  hl= [config.hidden_layer_size for h in range(config.hidden_layers)]\r\n",
        "  ffnn= FFNNetwork(x_train.shape[2], output_size= y_train.shape[1], hidden_layers=hl, act_func=config.activation, init_func=config.initialisation)\r\n",
        "  ffnn.fit(x_train, y_train, display_loss=True, opt_algo=config.optimizer, weight_decay=config.weight_decay, epochs=config.epochs, lr=config.learning_rate, batch_size=config.batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_udei3s-ifor"
      },
      "source": [
        "wandb.agent(sweep_id, train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oZwxuWBN-R7P"
      },
      "source": [
        "# Method for preprocessing image dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EmfWKuVdrcI3"
      },
      "source": [
        "def load_fashion_mnist_data(split_val=False):\r\n",
        "\r\n",
        "    \"\"\" Prepares image data for muti-class classification model \"\"\"\r\n",
        "\r\n",
        "    # Load data\r\n",
        "    (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()\r\n",
        "\r\n",
        "    # Flaten image data\r\n",
        "    x_train= x_train.reshape(x_train.shape[0], 1, x_train.shape[1] * x_train.shape[2])\r\n",
        "    x_test= x_test.reshape(x_test.shape[0], 1, x_test.shape[1] * x_test.shape[2])\r\n",
        "\r\n",
        "    # Convert from integers to floats\r\n",
        "    x_train_f = x_train.astype(np.float64)\r\n",
        "    x_test_f = x_test.astype(np.float64)\r\n",
        "\r\n",
        "    # Normalize\r\n",
        "    x_train_norm= x_train_f/255.0\r\n",
        "    x_test_norm= x_test_f/255.0\r\n",
        "\r\n",
        "    # Encode output labels using one hot encoding\r\n",
        "    encoder=OneHotEncoder()\r\n",
        "    y_train_encoded= encoder.fit_transform(y_train.reshape(-1, 1)).toarray()\r\n",
        "    y_test_encoded= encoder.transform(y_test.reshape(-1, 1)).toarray()\r\n",
        "\r\n",
        "    if split_val:\r\n",
        "      x_train, x_val, y_train, y_val= train_test_split(x_train, y_train, test_size=0.1, random_state=1)\r\n",
        "      return x_train_norm, x_val, x_test_norm, y_train_encoded, y_val, y_test_encoded\r\n",
        "    else:\r\n",
        "      return x_train_norm, x_test_norm, y_train_encoded, y_test_encoded"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NFQTGWwZQqvQ"
      },
      "source": [
        "def load_mnist_data(split_val=False):\r\n",
        "\r\n",
        "    \"\"\" Prepares image data for muti-class classification model \"\"\"\r\n",
        "\r\n",
        "    # Load data\r\n",
        "    (x_train, y_train), (x_test, y_test) = mnist.load_data()\r\n",
        "\r\n",
        "    # Flaten image data\r\n",
        "    x_train= x_train.reshape(x_train.shape[0], 1, x_train.shape[1] * x_train.shape[2])\r\n",
        "    x_test= x_test.reshape(x_test.shape[0], 1, x_test.shape[1] * x_test.shape[2])\r\n",
        "\r\n",
        "    # Convert from integers to floats\r\n",
        "    x_train_f = x_train.astype(np.float64)\r\n",
        "    x_test_f = x_test.astype(np.float64)\r\n",
        "\r\n",
        "    # Normalize\r\n",
        "    x_train_norm= x_train_f/255.0\r\n",
        "    x_test_norm= x_test_f/255.0\r\n",
        "\r\n",
        "    # Encode output labels using one hot encoding\r\n",
        "    encoder=OneHotEncoder()\r\n",
        "    y_train_encoded= encoder.fit_transform(y_train.reshape(-1, 1)).toarray()\r\n",
        "    y_test_encoded= encoder.transform(y_test.reshape(-1, 1)).toarray()\r\n",
        "\r\n",
        "    if split_val:\r\n",
        "      x_train, x_val, y_train, y_val= train_test_split(x_train, y_train, test_size=0.1, random_state=1)\r\n",
        "      return x_train_norm, x_val, x_test_norm, y_train_encoded, y_val, y_test_encoded\r\n",
        "    else:\r\n",
        "      return x_train_norm, x_test_norm, y_train_encoded, y_test_encoded"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "23oHRcfISqnm"
      },
      "source": [
        "# Training and Testing Model for fashion_mnist dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2HJbSO8BTGGH"
      },
      "source": [
        "## Loading Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NaLBDIcnrocn"
      },
      "source": [
        "x_train, x_test, y_train, y_test= load_fashion_mnist_data()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SL6v6j_tTJcJ"
      },
      "source": [
        "## Training Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_0KSJPIyrrMW"
      },
      "source": [
        "ffnn=FFNNetwork(x_train.shape[2], output_size= y_train.shape[1], hidden_layers=[128, 128, 128], act_func='tanh', loss_func='ce')\r\n",
        "ffnn.fit(x_train, y_train, weight_decay=0.0005, display_loss=True, display_accuracy=True, batch_size=32, opt_algo='rmsprop', epochs=10, lr=0.0001)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7YdMbHsJToKP"
      },
      "source": [
        "## Testing Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BqnqOSEUrur1",
        "outputId": "1800f317-e374-426d-f5ac-769e9972000f"
      },
      "source": [
        "y_preds=ffnn.predict(x_test)\r\n",
        "y_preds=np.argmax(y_preds, axis=1)\r\n",
        "test=np.argmax(y_test, axis=1)\r\n",
        "accuracy_score(y_preds, test)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8481"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AV4vLM0kS6i6"
      },
      "source": [
        "# Training and Testing Model for mnist dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SW29rYFoTOLg"
      },
      "source": [
        "## Loading Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LQmf0oEsTDLS"
      },
      "source": [
        "x_train, x_test, y_train, y_test= load_mnist_data()"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NI4sK-puTYfT"
      },
      "source": [
        "## Training Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O-ugaPngTcab"
      },
      "source": [
        "ffnn=FFNNetwork(x_train.shape[2], output_size= y_train.shape[1], hidden_layers=[128, 128, 128], act_func='relu', loss_func='ce')\r\n",
        "ffnn.fit(x_train, y_train, weight_decay=0.0005, display_loss=True, display_accuracy=True, batch_size=64, opt_algo='momentum', epochs=10, lr=0.001)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rHbIgjB9ThdR"
      },
      "source": [
        "## Testing Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ActzKOlETkEQ",
        "outputId": "1a95dbe8-c4c1-4ee5-dce1-663e08132e67"
      },
      "source": [
        "y_preds=ffnn.predict(x_test)\r\n",
        "y_preds=np.argmax(y_preds, axis=1)\r\n",
        "test=np.argmax(y_test, axis=1)\r\n",
        "accuracy_score(y_preds, test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8267"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CAd6Yir4ZfKE"
      },
      "source": [
        "# Plotting Confusion Matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZBxfltquZiss"
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\r\n",
        "import seaborn as sn"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 309
        },
        "id": "MSSMLlzvZpZ3",
        "outputId": "a03f7f0e-354a-4f0a-c526-8f1fad4e4e1e"
      },
      "source": [
        "labels=[\"T-shirt/top\",\"Trouser\",\"Pullover\",\"Dress\",\"Coat\",\r\n",
        "        \"Sandal\",\"Shirt\",\"Sneaker\",\"Bag\",\"Ankle boot\"]\r\n",
        "cm=confusion_matrix(test, y_preds)\r\n",
        "figure=sn.heatmap(cm, annot=False, xticklabels=labels, yticklabels=labels, cmap='Greens')\r\n",
        "wandb.log({ 'confusion_matrix': wandb.Image(figure) })\r\n",
        "# wandb.log({\"conf_mat\" : wandb.plot.confusion_matrix(probs=None, y_true=test, preds=y_preds, class_names=labels, title='Confusion Matrix')})\r\n",
        "# wandb.log({'confusion_matrix_heatmap': wandb.plots.HeatMap(labels, labels, cm, show_text=False)})"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEkCAYAAADeqh2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxcVbnu8d/THSABZJJBBCSIkdEQICIgIINwRSYZZBA9oBwj5yhOVwXFI6AeBXE4ggoGAwRlBrmAKLMgIFOAkAQZ5DAGQWSekkCS9/6xVpNKU91V3b33rqrO8+VTn65atWu/u5pOvbVmRQRmZmb96Wr1BZiZWftzsjAzs4acLMzMrCEnCzMza8jJwszMGnKyMDOzhka0+gLayfJHbl7ZOOInv3ttVaGYH/MqijO/kjgAI7oWqyxWlZ547bFK4qy25LsqiQMQVDc8X6iyWCO7lxxyMO24etO/nLhqZnVvrg4nCzOzVlFLP/8HxMnCzKxVOqgjwMnCzKxVXLMwM7OGOidXOFmYmbWMaxZmZtbQcOyzkPR24Jr88B3APOBf+fFmEfF6H68bDfwhIjas89x3gb9ExNV1njsYuDIi/lFTtj+wNnAT8HpE/LXZ6zczaztdw7BmERHPAuMAJB0NvBIRPx5K8Ij4Tr1ySd3AwcAM4B81T+0MnADsBrwCOFmYWefqoGRRaCVI0gaSbpM0VdI0SWPyU92STpF0j6QrJY3Kx58uaZ98/xFJx0m6EzgAGA+cmc81SpJIyeo54FDgK/m5rSWNlnRtjnmNpHfVnP9kSVMkPSBp1yLfr5nZkGgAtxYrusXsUODnETGO9GE/M5ePAX4ZERsALwB79/H6ZyNik4j4HTAFODAixkXELGBj4O6IeBg4GfhZfu4G4ERgckSMBc4k1T56jAY2A3YBTpY0ssD3a2Y2eFLztxYrOlncDHxL0uHAmvlDHuDhiJia799B+gCv59x+zv0R4E99PLcFcFa+/1tgq5rnzouI+RHxd+AhYN3aF0qakGseU+bc9XQ/4c3MCrao1Cwk7ZmbgqZKGh8RZwG7A7OAP0raPh86p+Zl8+i7r+TVfsLtBFw5iMvsvfbKQo8jYmJEjI+I8UtsvPIgTm9mNkjdav7WYkNKFhFxUW4KGhcRUyS9G3goIk4ALgbGDuH0LwNvA5C0LDAid7Iv9Fz2V2D/fP9A4Iaa5z4uqUvS2sC7gfuHcE1mZsVZhJuh9gVmSJoKbAicMYRznU7qY5hKqq3UDq+9FOip1WwNHAZ8WtI04FPAl2qOfQy4jdSEdWhEzB7CNZmZFadLzd9aTBHVLR88WJJ+A/wmIm4Z4OtOJ83xuKCZ471E+VDjeInyofIS5UPTcUuUHzim+SXKz/y7lyhvJCL+vdXXYGZWuDZoXmpWRySLwYqIg1t9DWZmfeqcXDG8k4WZWVtrg1FOzXKyMDNrFTdDmZlZQ8Nx1dlFQZUjlEZ95L2VxZp1+QOVxKly1MtwHWHzziXXqCxWVar8/XUc1yzMzKyhzskVThZmZi3jmoWZmTXk0VBmZtZQ5+QKJwszs5ZpgzWfmuVkYWbWKu6zMDOzhjonV5SfLCS9HbgmP3wHafOjf+XHm0XE62Vfg5lZO+rq6pxZeaUni7xh0TgASUcDr0TEj3uelzQiIuaWfR018bojKlqz28ysHx3UCtWayeaSTpd0sqRbgR9JGifpFknTJF0kafl83HWSxuf7K0p6JN/fQNJtefOjaZLG5PJP1pT/WlJ3Ln9F0k8k3U3ar9vMrOW6pKZvrdbKOtDqwJYR8VXSjnqHR8RYYDpwVIPXHgr8PCLGAeOBmZLWA/YDPpjL55G2WAVYCrg1IjaKiBtLeC9mZgMmqelbq7UyWZwfEfPy/trLRcT1uXwysE2D194MfEvS4cCaETEL2AHYFLg9b8W6A2nPbUiJ48J6J5I0QdIUSVMmnXLqEN+SmVnzik4Wkr4i6R5JMySdLWmkpLUk3SrpQUnnSlo8H7tEfvxgfn50f+du5WioV5s4Zi4LEtrInsKIOCs3Ye0C/FHS50jjCiZHxDfrnGd2X/0UETERmAgwe95r7b/HrJkNG0XWGCStBnwRWD8iZkk6D9gf+Cjws4g4R9LJwCHASfnn8xHxHkn7A8eRWmfqanlXfES8CDwvaetc9Cmgp5bxCKm2ALBPz2skvRt4KCJOAC4GxpJGXO0jaeV8zAqS1iz/HZiZDU5Xl5q+NWkEMErSCGBJ4Elge+CC/Pxk4GP5/h75Mfn5HdRP9mp5ssgOAo6XNI00cuq7ufzHwH9IugtYseb4fYEZublpQ+CMiPgb8G3gynyeq4BVq3oDZmYDVWQzVEQ8QfrMfIyUJF4E7gBeqBlxOhNYLd9fDXg8v3ZuPv7tfZ2/0maoiDi6j/KpwOZ1yu8j1Rp6fDuXHwscW+f4c4Fz65QvPbgrNjMrz0D2+pA0AZhQUzQxN6P3PL88qbawFvACcD7wkWKu1DO4zcxaZiB9FrX9q334MPBwRPwrn/v3wAeB5Wrms60OPJGPfwJYgzSadASwLPBsXydvl2YoM7NFjtT8rQmPAZtLWjL3PewA/A34Mwv6fA8i9fMCXJIfk5+/NiL6HOTjmoWZWYsUOdkuIm6VdAFwJ2kk6V2kmshlwDmSvp/LJuWXTAJ+K+lB4DnSyKk+OVmYmbVI0WtDRcRRvHVS80PAZnWOnQ18vNlzO1mYmbVIG0zMbpqTRY35Fa4v+Nrl91cWa/VjPlxJnMePuqqSOAAvvf5CZbGWXXz5ymK9/MaLlcRZZrHlKokDEFQ313Ugo4vaQTss49EsJwszsxZxsjAzs4acLMzMrKEBLOPRck4WZmYt4pqFmZk15GRhZmYNtcMOeM1ysjAza5EOyhXlrA0laV7eB3uGpPMlLdng+Nq9th+RtGJ/x5uZDQfeVhVmRcS4iNgQeJ20Z3bLKfHiiWbWFrrU1fSt1aq4ghuA90jaVtIfegol/ULSwf29UNJXc+1khqQv57JjJX2+5pijJX0t3/+6pNslTZN0TC4bLel+SWcAM0hL8pqZtZxrFlleI31nYPogXrsp8GngA6SNkT4raWPS5kb71hy6L3CupJ2AMaQFs8YBm0raJh8zBvhVRGwQEY/2ijNB0hRJU0495bSBXqaZ2aAVvER5qcrq4B6VtzyFVLOYBGw5wHNsBVwUEa/Cmxt5bB0RJ0haWdI7gZVIG44/LulLwE6kJXgBliYliceARyPilnpBajcUeW3uy9UtYmNmi7x2qDE0q6xkMSsixtUWSJrLwjWZkUM4//mkzTrewYJtVAX8MCJ+3SvuaODVIcQyMytFJyWLKntNHgXWl7SEpOVIuzj15wbgY3nXp6WAPXMZpASxPylhnJ/LrgA+I2lpAEmrSVq56DdhZlaUTuqzqGyeRW4qOo/UyfwwC5qL+jr+TkmnA7flot9ExF35uXskvQ14IiKezGVXSloPuDn/Yl8BPglUt+64mdkALPJrQ0XE0n2UfwP4Rp3ybWvuj665/1Pgp32c6311yn4O/LzO4Rs2umYzs6q1Q42hWZ7BbWbWIk4WZmbWUAflCicLM7NWcc3CzMwa6upq/TIezXKyMDNrkQ6qWDhZ1AoqnMAd1cV69DuXVxJnhSO2aXxQQZ7+wbWVxaqS6KBPjyYNx/dUFDdDmZlZY04WZmbWiGsWZmbWUAflCicLM7NW8WgoMzNryM1QZmbWUAflivZIFpLmkXbTWwyYC5wB/Cwi5rf0wszMSuSaxcC9uVlS3oPiLGAZ4KjagySNiIi5Lbg+M7PCdVKyaLvelYh4GpgAfEHJwZIukXQtcI2kpSSdKuk2SXdJ2gNA0ga5bKqkaZLG5GMvk3S3pBmS9mvpmzMzq+HNj4YoIh6S1A307HS3CTA2Ip6T9APg2oj4TN5x7zZJVwOHAj+PiDMlLQ50Ax8F/hERuwBIWrb6d2NmVl8nbX7UdjWLPlwVEc/l+zsBR0iaClxH2sv7XcDNwLckHQ6sGRGzSP0gO0o6TtLWEfFi7xNLmiBpiqQpp55yWiVvxswMXLMYMknvJm2H+nQuerX2aWDviLi/18vulXQrsAvwR0mfi4hrJW1CqmF8X9I1EfHd2hdFxERgIsCrc1+qcHEoM1vUtUMSaFbb1SwkrQScDPwiou5qe1cAhyn/liVtnH++G3goIk4ALgbGSnon8FpE/A44ntScZWbWFoquWUhaTtIFku6TdK+kLSStIOkqSX/PP5fPx0rSCZIezP28/X4+tkuyGJU7pu8BrgauBI7p49jvkYbYTsvHfy+X7wvMyM1TG5KG376P1KcxlTSy6vslvgczswGRmr816efA5RGxLrARcC9wBHBNRIwBrsmPAXYGxuTbBOCk/k7cFs1QEdHdz3OnA6fXPJ4FfK7OcccCx/YqviLfzMzajgpc7iMP4NkGOBggIl4HXs8jRrfNh00m9fUeDuwBnJFbcG7JtZJVI+LJeudvl5qFmdkip+BmqLWAfwGn5WkFv5G0FLBKTQJ4Clgl318NeLzm9TNzWV1OFmZmLdKl5m+1IzfzbUKv040g9cueFBEbkwYGHVF7QK5FDGogT1s0Q5mZLYoGMhqqduRmH2YCMyPi1vz4AlKy+GdP85KkVVkwyvQJYI2a16+ey+pyzcLMrEW6pKZvjUTEU8DjktbJRTsAfwMuAQ7KZQeRRouSy/8tj4raHHixr/4KcM3CzKxlSphncRjQs4rFQ8CnSZWC8yQdAjxKGjkK8EfSHLQHgdfysX1ysjAza5ERBSeLiJgKjK/z1A51jg3g882e28miRreG56/juTnPNT6oAM/88M+VxAFY7/g9Kov1wDcuqyzW3PlvVBbLWq+TZnAPz09HM7MO0ExfRLtwsjAzaxHXLMzMrKFOGo7qZGFm1iLdBS73UTYnCzOzFnGfhZmZNdQ5qaLNm8wkvUPSOZL+V9Idkv4o6b0DPMdykv6zrGs0MxusImdwl36trb6AvuTNjS4CrouItSNiU+CbLFgxsVnLAU4WZtZ2nCyKsR3wRkSc3FMQEXcDN0o6XtIMSdMl7QcgaWlJ10i6M5f3zNo6Flg7b650fPVvw8ysPu/BXYwNgTvqlO8FjCPtArUicLukv5DWcd8zIl6StCJpM49LSKsubhgR4yq6bjOzpnS3QRJoVjvXLPqyFXB2RMyLiH8C1wPvJ/UV/UDSNNLWrKvRRJNV7Rrxk045tczrNjNbSCc1Q7VzzeIeYJ8BHH8gsBKwaUS8IekRYGSjF9WuET973muD2hTEzGww2iEJNKudaxbXAkvU7gYlaSzwArCfpG5JK5H2nL0NWBZ4OieK7YA188teBt5W7aWbmTXmPosCRERI2hP4H0mHA7OBR4AvA0sDd5O2B/xGRDwl6UzgUknTgSnAffk8z0q6SdIM4E8R8fUWvB0zs7fopJpF2yYLgIj4Bws26qj19XyrPfYZYIs+zvOJ4q/OzGxoOidVtHmyMDMbzkZ4bSgzM2ukHfoimuVkYWbWIp1Tr3CyMDNrGdcszMysIY+GsraywhIrtvoSCvfANy6rLNaoA95XWaxZZ0+vJM6cebMriQOwRHfDubGFeX3+nMpijexecsjn8OZHZmbWUFcHDZ51sjAzaxH3WZiZWUPuszAzs4bkZigzM2vEzVBmZtZQt7pbfQlNc7IwM2uRTuqzqHyQr6QjJd0jaVreF/sDBZzzOknjh3qMmVmVvJ9FHyRtAewKbBIRc/Je2YtXeQ1mZu3CHdx9WxV4JiLmwJt7UCDpO8BuwCjgr8Dn8uZH1wG3AtsBywGHRMQNkkYBpwEbkTY5GtUTQNJJpD25RwEXRMRRFb03M7MBcTNU364E1pD0gKRfSfpQLv9FRLw/IjYkfcjvWvOaERGxGWmHvJ4P/v8AXouI9XLZpjXHHxkR44GxwIfyVqxmZm2nk5qhKk0WEfEK6YN9AvAv4FxJBwPbSbo1b4m6PbBBzct+n3/eAYzO97cBfpfPOQ2YVnP8vpLuBO7K51m/v2uSNEHSFElTJp1y6hDenZnZwHSru+lbq1U+Gioi5gHXAdfl5PA5Ui1gfEQ8LulooHblsZ6VwebR4HolrQV8DXh/RDwv6fRe56p3PROBiQCz570WA30/ZmaD1Q41hmZVWrOQtI6kMTVF44D78/1nJC0N7NPEqf4CfCKfc0NSsgFYBngVeFHSKsDOhVy4mVkJulDTt1arumaxNHCipOWAucCDpCapF4AZwFPA7U2c5yTgNEn3AveSmqiIiLsl3UXq9H4cuKnwd2BmVpCiaxaSuoEpwBMRsWtubTkHeDvpc/JTEfG6pCWAM0jdAs8C+0XEI/2du9JkERF3AFvWeerb+db7+G1r7j9D7rOIiFnA/n3EOLiP8m3rlZuZtUoJo6G+RPoCvUx+fBzws4g4R9LJwCGkL9uHAM9HxHsk7Z+P26/fay36Ss3MrDlFdnBLWh3YBfhNfizSgKEL8iGTgY/l+3vkx+Tnd1CDao6ThZlZixQ8dPZ/gG8A8/PjtwMvRMTc/HgmsFq+vxqpqZ78/Iv5+D45WZiZtYgG8l/NMP98m/DmeaRdgadzU38pvJCgmVmLDKTPonaYfx0fBHaX9FHSdIFlgJ8Dy0kakWsPqwNP5OOfANYAZkoaASxL6uju+1qbvlIzMyvUQGoW/YmIb0bE6hExmjT459qIOBD4MwumIxwEXJzvX5Ifk5+/NiL6nWfmmkWNF17vN7EWarnF+20eLNRzc56pJM6yiy9fSRyAN+a/XlmsWWdPryzW5qccUEmcWz57diVxAObHvMpiLd61RGWxilDB2lCHA+dI+j5pVYtJuXwS8FtJDwLP0cfo0lpOFmZmLdJVwjIeEXEdaZUMIuIhYLM6x8wGPj6Q8zpZmJm1SCetOutkYWbWIp20NpSThZlZi7TDmk/NcrIwM2sR1yzMzKwhqXNmLzhZmJm1SHcHJYu2vlJJR0q6R9I0SVMlfUDSI5JWrHPs7pKO6OM820qqt9qtmVnLFDUprwptW7OQtAVpL+5NImJOThCL93V8RFxCmpXY+zwjgG2BV4C/lnO1ZmYD5z6LYqwKPBMRc+DN/Sx6frmHSdoNWAz4eETcl/fyHh8RX8jbqc4GNiatgbIlME/SJ4HDIuKGqt+MmVlvnTQaqp2boa4E1pD0gKRfSfpQzXPPRMQmpE08vtbH61cHtoyIvYCTSRuAjOudKGpXcvztpDPLeB9mZnVJXU3fWq1taxYR8YqkTYGtge2Ac2v6JH6ff94B7NXHKc6PaLwoTe1Kjk/NerzfhbTMzIrU1QZJoFltmywA8of9dcB1kqazYJXEOfnnPPp+D6+We3VmZkPTDh3XzWrbtCZpHUljaorGAY8O8nQvA28b+lWZmRWn4J3yStW2yQJYGpgs6W+SpgHrA0cP8lyXAnvm4bdbF3WBZmZD0YWavrVa2zZD5e0B682NGF1zzBTSsFgi4nTg9Hz/4F7negAYW8Z1mpkNVjvUGJrVtsnCzGy4U1s37izMycLMrEU8GsrMzBry5kdmZtZQJw2ddbIwM2sRd3B3qOUWf3urL6EUKyzxlkV6O15396jKYs2d/0ZlsW757NmVxBm1yzqVxAGYddn9lcWa13jRhrbiDm4zM2uok/azcLIwM2sRN0OZmVlD7uA2M7OGXLMwM7OG3MFtZmYNddKkvNLSmqQjJd0jaVpe7fUDBZ57W0l/KOp8Zmat0KWupm+tVkrNQtIWwK7AJhExR9KKwOJlxBooSSMiYm6rr8PMrJM6uMtKV6uS9smeAxARz0TEPyQ9IukYSXdKmi5pXQBJS0k6VdJtku6StEcuHy3phnz8nZLesmS5pPfn16wtaVNJ10u6Q9IVklbNx1wn6X8kTQG+VNJ7NjMbEG9+BFcCa0h6QNKvJH2o5rlnImIT4CTga7nsSODaiNiMtN/28ZKWAp4GdszH7wecUBskJ4+TgT2Ax4ATgX0iYlPgVOC/aw5fPCLGR8RPin6zZmaD0TWA/1qtlCuIiFeATYEJwL+AcyUdnJ/+ff55Bws2MtoJOELSVNKe2yOBdwGLAafk/bfPJ+2W12M9YCKwW0Q8BqwDbAhclc/zbWD1muPPrXetkiZImiJpyqRTTh3sWzYzG7BOqlmUNhoqIuaRPvivyx/2B+Wn5uSf82riC9g7IhZaREbS0cA/gY1IiW12zdNPkpLKxsA/8jnuiYgt+rikV/u4zomkpMPsea9Fc+/OzGzoFvk+C0nrSBpTUzQOeLSfl1wBHKacPiVtnMuXBZ6MiPnAp4Dumte8AOwC/FDStsD9wEq5cx1Ji0naoIj3Y2ZWhk4aDVXWFSwNTJb0N0nTSM1HR/dz/PdITU7TJN2THwP8CjhI0t3AuvSqHUTEP0mjrn5JqmHsAxyXj59K/T28zczaggbwX6spwi0vPdwMZfVUuUT5iK7FKonjJcqHbqkRbxvyJ/iUZ/7a9GfO+BW3bGnGaH3dxsxsEVVkzULSGpL+nFt07pH0pVy+gqSrJP09/1w+l0vSCZIezJOnN+nv/E4WZmYtUvBoqLnA/42I9YHNgc9LWh84ArgmIsYA1+THADsDY/JtAmk6Q5+cLMzMWqTIeRYR8WRE3JnvvwzcC6xGmoc2OR82GfhYvr8HcEYktwDL9UxkrscLCZqZtUhZ8yckjSYN+rkVWCUinsxPPQWsku+vBjxe87KZuexJ6nDNwsysRQbSZ1E7gTjfJtQ9p7Q0cCHw5Yh4qfa5SCOaBjWQxzULM7MWGciQ2NoJxH2eT1qMlCjOjIie1TL+KWnViHgyNzM9ncufANaoefnquawuJ4saMbiEOyhVjpuuajhhlROHqvz9VTWcFWB+zK8kzmuX3VdJHIBRu69XWaxZl9xbWawiFNkMlSc1TwLujYif1jx1CWkFjWPzz4tryr8g6RzgA8CLNc1Vb+FkYWbWIgV/6fkgaaWL6Xl9PIBvkZLEeZIOIa2ksW9+7o/AR4EHgdeAT/d3cicLM7MWKbI2HhE3Qp/ZZ4c6xwfw+WbP72RhZtYi7bCMR7OcLMzMWqQdlh5vlpOFmVmLuGZhZmYNOVmUQNI8YDqpA2ce8IWI+Gtrr8rMbPDcDFWOWRExDkDS/wF+CHyo/5eYmbUvtcGmRs3qnCtd2DLA85Cmtku6RtKdkqZL2qPnIEn/Jel+STdKOlvS11p2xWZmvXTS5kedVLMYlSeajARWBbbP5bOBPSPiJUkrArdIugQYD+xN2r97MeBO4I7qL9vMrL52SALN6qSaxayIGBcR6wIfAc7I09sF/CBv33o1adXEVUizGS+OiNl5ud5L6520dnGuSaecWs07MTOj8P0sStVJNYs3RcTNuRaxEmm6+krAphHxhqRHSLWPZs/15uJcs+a96m1VzawyrlmUTNK6QDfwLLAs8HROFNsBa+bDbgJ2kzQyL9m7a2uu1sysvi51NX1rtU6qWfT0WUBqejooIuZJOhO4VNJ0YApwH0BE3J77LqYB/yQNu32xBddtZlZXJ9UsOiZZRER3H+XPAFv08bIfR8TRkpYE/oI7uM2sjbRDX0SzOiZZDNLEvGH5SGByz/60ZmbtwDWLNhERn2j1NZiZ9c3JwszMGuicVOFkYWbWMp203IeThZlZi7hm0aE6qbNpILpVdyCZNWl+zK8sVlXj6WfPm1VJHIBZl9xbWaxRe61fWay4+JECztI5nzlOFmZmLdJJQ2c7p8HMzMxaxjULM7MW6aSmbycLM7MWcbIwM7OG3GdhZmbDimsWZmYt0knNUE3VLCR9TFLkfSSaOf6RvDlR7/JXBnJxAz2+n/McLOmdRZzLzKw4GsCttZpthjoAuDH/7EQHA04WZtZWuqSmb63WMFnkXea2Ag4B9q8p31bSdZIukHSfpDPVq7dG0ihJf5L02Trn/bqk2yVNk3RMP/F/JukeSddIWimXjZN0S37tRZKW76tc0j7AeOBMSVMljWryd2NmVrLhVbPYA7g8Ih4AnpW0ac1zGwNfBtYH3g18sOa5pYFLgbMj4pTaE0raCRgDbAaMAzaVtE2d2EsBUyJiA+B64KhcfgZweESMJe2A12d5RFxA2kHvwIgYFxHVrXNgZtaPzkkVzSWLA4Bz8v1zWLgp6raImBkR84GpwOia5y4GTouIM+qcc6d8uwu4E1iXlDx6mw+cm+//DthK0rLAchFxfS6fDGzTV3mjNydpgqQpkqZMOuXURoebmRWoc9JFv6OhJK0AbA+8T1IA3UBI+no+ZE7N4fN6ne8m4COSzoqI6H1q4IcR8esBXm/v8wxZREwEJgLMnvda4ec3M+vLcJpnsQ/w24hYMyJGR8QawMPA1k2c+zvA88Av6zx3BfCZ3B+CpNUkrdzH9e2T738CuDEiXgSel9RzDZ8Cru+rPN9/GXhbE9dsZmZ1NEoWBwAX9Sq7kOZHRX0JGCXpR7WFEXElcBZws6TpwAXU/zB/FdhM0gxSDee7ufwg4HhJ00h9Ho3KTwdOdge3mbUT0dX0rdX01haiRZeboawe72cxNCO7q/t+VvF+FkNuQ3rljReb/sxZerFlW9pm5RncZmYt0kl9Fk4WZmYt0znJovUNYWZmi6iiB85K+oik+yU9KOmIIq/VNQszsxZRgX1UkrpJo093BGYCt0u6JCL+VsT5XbMwMxseNgMejIiHIuJ10iTqPYo6uZOFmVmLaAD/NWE14PGaxzNzWSHcDFVjZPeSg+ptkjQhzwQvXVWxhuN7cqwFRnYvWUmcoRhMrLj4kcpiFWEgnzmSJgATaoomVnnNrlkUY0LjQzou1nB8T47VOXGGc6xBiYiJETG+5tY7UTwBrFHzePVcVggnCzOz4eF2YIyktSQtTtpS4pKiTu5mKDOzYSAi5kr6AmntvW7g1Ii4p6jzO1kUo8q2zqpiDcf35FidE2c4xypNRPwR+GMZ5/baUGZm1pD7LMzMrCEni0GStLiksZLelzuTyojRJWnLMs5ttiiTtEQzZbaAk8UgSNoF+F/gBOAXwIOSdi46Tt6utt7mUYVTskbjI60dSPpgM2UFxOmW9JWiz9sGbm6yzDL3WQyCpPuAXSPiwfx4beCyiFi3hFg/Jv0R/77O9rRFx5oeEe8rM0ZNrI8Dl0fEy5K+DWwCfD8i7iw4zj+hVPEAABEDSURBVHERcXijsiHG2KS/54t+TznmnRGxSaOygmLdFhGbFX3ePmJ9tU7xi8AdETG1gPO/gzSr+Xek3Td7JsUtA5xcxr/h4cLJYhAk3R4R7695LOC22rICY70MLEXa43wW6Y87ImKZEmJNBn4REbcXfe46saZFxFhJWwHfB44HvhMRHyg4Tr0P1WkRMbbAGH/u5+mIiO0LjLUFsCXwZeBnNU8tA+wZERsVFasm5s+AxYBzSbtXAqUlwbOA8cCluWhXYBowGjg/In7Ux0ubPf9BwME5xpSap14GTo+I3w/l/MOZk8UgSDoJWBM4Dwjg48BjwNUAnfoHl2tM7wEeJX0o9CSmwj5Ya2LdFREbS/ohMD0izuopK+j8/wH8J/BuUpNhj7cBN0XEJ4uIUzVJHwK2BQ4FTq556mXg0oj4ewkx6yXDQpNgTay/AB+NiFfy46WBy4CPkGoXhWyFJ2nviLiwiHMtKpwsBkHSaf08HRHxmQJjCTgQWCsivpf7FVaNiNuKilETa8165RHxaAmx/kBaimBHUhPULFLtrJBvxpKWBZYHfgjUruv/ckQ8V0SMPuJuCKwPjOwpi4gzCo7RDZwXEXsXed52kL+wvC8i3siPlwDujoh1C/4ysRzwHWCbXHQ98N2IeLGI8w9HThZtLtdi5gPbR8R6kpYHriyjySvH2woYExGnSVoJWDoiHi4hzpKkb4vTI+LvklYlfUhcWXSsHG9lFv4Af6yEGEeRvvWvT5oYtTNwY0TsU0KsmyNii6LP20esVYAfAO+MiJ0lrQ9sERGTSoj1X8CewMW5aDfSkhU/IS2cd2BBcS4EZgCTc9GngI0iYq8izj8cOVkMgqTVgROBntEnNwBfioiZJcS6MyI2qf1WJenuktqmjyK15a4TEe+V9E5SO3EZo2zWBmZGxBxJ2wJjgTMi4oWC4+wG/BR4J/A0qfnw3ojYoMg4OdZ0YCPgrojYKH/I/i4idiwh1kmkjtrzWbgfofAmUEl/Ak4DjszvawTpPZYyGELS+0n9MpCaDKf0d/wgY0yNiHGNymwBD50dnNNI33bemW+X5rIyvJGbHQIgf9ufX1KsPYHdyR8+EfEPUht/GS4E5kl6D2mphTWAs0qI831gc+CBiFgL2AG4pYQ4ALPycOe5kpYhJaeyhiOPBJ4Ftid9+96N1BlchhUj4jzy311EzCUNuChFHmBxNnAR8LSkd5UQZlauRQNvDjueVUKcYcNrQw3OShFRmxxOl/TlkmKdQPpHs7Kk/wb2Ab5dUqzXIyIk9SSmpUqKAzA/L3y2F3BiRJwo6a4S4rwREc/mCY5dEfFnSf9TQhyAKbkt/BTgDuAVShq7HxGfLuO8fXhV0ttZ8IVlc9Jw1sJJ2p3U5NRTE3wXcB9QdE3wP4DJuW9LwHPAQQXHGFacLAbnWUmfJH37ATiA9C2vcBFxpqQ7SN+IBXwsIu4tIxZwnqRfA8tJ+izwGdIHXxnekHQA8G+kb8WQhmcW7YU8ouYG4ExJT1PTbFOkiPjPfPdkSZcDy0TEtCJjSPpGRPxI0onkD+9e1/DFIuNlXyXVpNeWdBOwEulLSxm+R6oJXp1Hy20HFD5yLc/Z2CjXAImIl4qOMdy4z2IQ8qihE4EtSP9g/wocFhGP9/vCwcWqpG2/Jt6OwE6kxHRFRFxVUpz1ScM/b46IsyWtBewbEccVHGcpYDbp/RwILAucGRGFJfcqJ+VJ2i0iLs3zBerFmlyvfIgx1yJt17kO6fd4PzCujPk4kqZExHhJdwMbR8T8Mvroco3iKDwaqmlOFoMg6YMRcVOjsoJiTSV1Oo8mjTe/BNggIj5aQqyvAudGRGG7azWINwp4V0TcX3KcVYCe0WO3RcTTBZ+/Zx7CSNL/q7tJH6pjgSlVjVoqS67Z7t7zdyFpG+CXZXRwS7oa+BhpyPOKpKao90dEoWukeTTUwLmDe3BObLKsCPNzh+JepNnVXwdWLSnW24ArJd0g6Qv5Q7YUeZTSVODy/HicpMJ29aqJsy9wG2ni5L7ArZIKbUKJiO0iYjvgSWCTSFtebgpsTIHbWtaS9F5JEyVdKenanlsZsUg1wP8n6R2SPkr6Wy/8y0q2B/Aa8BXS38b/sqCZskhrR8RREfFQvh1DmsBpfXCfxQBowVILK2nhNWyWIe1MVYaq2vbJ/2COkTQW2A+4XtLMiPhwCeGOBjYDrsuxp0oq4x/rkaRvpk/Dm6PJrgYuKCHWOhExvedBRMyQtF4JcSANmT0Z+A0ljkyCNDpJ0heBK0lNeh+OiH+VFKunP2m+pMuAZ6Oc5o9ZkraKiBvBo6Ga4WQxMIsDS5N+b7VDSl+ivA6/T5O+2f13RDyc249/W1KsHk8DT5E67VcuKcYbEfGipNqyMoYEd/VqdnqW8mrU0yT9hrRIHaQ+kkI7uGvMjYiTSjo3AJIuZeFO9CVJo6AmSSIidi8w1ubAsaRRSd8j/Y2vCHRJ+reIuLyoWJlHQw2Q+ywGQNK3gD8Bz5WxBEarSfpPUlPNSqRvrudFxN9KijUJuIa0FMfewBeBxSLi0ILjHE/qO+gZubYfMC0KXHW2JtZI0odQT6fpX4CTImJ2gTFWyHe/SErqFwFzep4vcikTpXWo+hQR1xcYawrwLdIAhInAzhFxi6R1gbOLWuajTlyPhmqSk8UASNqPtITDRqROzD+Rlt54vsSYD1N/iGThTTZKi/qdGwUsBd1ErCVJTUQ75aIrSEuUF/LBmif7rRIRN+W5HD0TsF4gjYb6375f3b5q/h56qmQL/W2U8XdRhdrZ05LujYj1ap4rbE2omnO+nTQaaivS7/BG0mioUobADwdOFoMkaWPS2kY7kforribtz1DoAn/5j7rHSFJH7QoR8Z0i49TE2wjYOj+8ISLuLiFGN2kc/XZFn7smxh+Ab9b2IeTy9wE/iIjCO01zu/fRpCVF3mziLfIDXNJmwOMR8WR+fBCpZvYIcHSRNYuamJuTOrXXIzXFdgOvRoHL5KtmKXn1Wla+9+OC4l1FqvnVNhluW1L/3LDgZDEIkpaIiDk1j5chLZOxTURMqCD+HXm0TdHn/SIwAehZX2hP0uJthY/0knQNsFdZ49rVa8+RXs+VssmT0oqpXyHN3n6z07ngOR13kjqYn8tDWM8BDgPGAetFOYsWTgH2JzVNjicNtnhvRHyzwBjzWLAs/ijSiCjy45ERUeigDkkzImLDXmWVbf7VidzBPTg3k5bVBlJ7p6SvFv3tB94y4auL9I+1rP9v/w58oGdEiqTjSO+1jGHBrwDT8ze82oXwipqBvFw/z40qKEZvL0bEn0o6d4/umtrDfqRkfiFwYZ6TU4qIeFBSd0TMA05TWpqlsGQREWWNJuzLlZL2J+1JA2mAyhUVX0NHcbIYAC3YknFUboaq3ZJxyZLC/qTm/lxSc8O+JcUSCw/DnMeC91i037OgBlOGKZI+GxELLVci6d9J3/zL8Ofcof57Fu50LnJHuW5JI/Lcmx1INcEeZf17fk3S4sBUST8izSfpyDlaSjtP9vT5fJkFIwu7SV9gvtaiS2t7boYaAC28JePtLPggfQmYHB26Q16PPHfkINIIG0gzaU+PiFIW3stzHihjzH6eUHgR8DoLksN4Upv7nhHxVAkxS99RTtKRpAlxz5AW2dskIiJ36E+OcpaTXxP4J+l39xXSiKVfRd6D3hYNThYDJKkLOCAizqwoXqVr2ORmr56RQzdERKErwSpNrDgK+ALp26lINaYTI+K7RcbK8bYDetqm74mIsmY5VyZ3OK9KGonX02T4XtJGVYXvi53PX8nSLNa+nCwGQXmxs4pilb6GTc3Y/boKHrv/VdLw4wmRd+DLM7dPIo0m+1lRsVpB0i6k5bRrd+UrPAlWKS/N8mNg8YhYS9I40heWwiblWftzshgESceSmgHOZeHO2TKGLZa+o1c/Y/dFakYpcujnXcCOEfFMr/KVSN+US5l8VQVJJ5P6rrYjLcOxD2nhwkNaemFDpLSQ4PbAdbFgt0aPHFrEuIN7cPbLPz9fUxaUsxBZ6WvYRNpBriqL9U4U+Rr+JamUNa8qtGVEjJU0LSKOkfQT0sTNTldvaZaO/5apivabHy6cLAah4g/XQ4Ezct8FwPMUvIaNKtyPgdThPJjnOkFPEn9Naf/y5yhvheAq3SPpE6SRWGNIS438tcXXNCSq2W+etCXyYqQJeoUPEBgunCwGQNL2EXFtXj7iLYoeDZVnOn8qIsre0esn/TwXpCaIomwkqd57EDXt/B3qD0rbqv6IBSOwftPC6ynKYaSlWeaQ1ti6grTYXyfbk7SE/J2Q9puXVNZ+88OCk8XAfAi4lvrr6wcFzhvoGUufq8qlLnRW5rIbdWJVPfmqdJLeT1qC43v58dLAdNLe0R3dYQ8QEa+RksWRrb6WAlW53/yw4A7uNtWzHo6kk0gTAc9n4c70wud0SPq3euURcUbRsYaTVizBUaU8LPdrpN0aa9e8KrLGWSlJXwPGADuSduX7DHBWGUvbDBeuWQyCpCVIi7eNZuF/PGUMkRxJ2oNhexaMWCq0FlOjdi2lkaQZwncCThb9a8kSHBWqbKOlqkTEj5X2m3+J1G/xnShpv/nhwslicC4mbQJzBzXLOhRs5TwnYQYLD2uFkkaiRMRhtY9z+/s5ZcQaZlqxBEeVSt9oqRVycnCCaNJw+ENuhdUj4iMlx+gm7cpXb22mqtoOXwWqHPnVqc4mbUH7DGlE1A3w5p4apcy0r9ilShtjlbbRUlV6rQ1V+++oZ05RYcuuDzfusxgESRNJy1NMb3jw4GMUvoZ/EzFrt9HsAtYn7ZZ3RJXX0YlasQRHVfKkTRgmGy3Z4DhZDICk6aR/MCNInWMPkb5p9XwrGVtgrMJ3B2siZu02mnOBRyNiZpXXYO2jZpTXU/lx6RstVUXSIRExqVfZsf5i1Dc3Qw3MrhXG2qGqQEp7Rx8KvIc05HNSbn+3RduvgQ8D5FFeP2TBKK+JpOVMOtXekmb3LAgq6ZeUt8/JsOBkMQAR8SiApLWBmRExR9K2wFgKHjFU8be2ycAbpLb2nUnNT1+qML61p+E8ymtv4BJJ80nbI78QEZ9p8TW1tY7cwKQNXAjMyx2YE4E1gLNae0lDsn5EfDIifk36trh1oxfYIqFbUs8Xyh1IE1J7dOQXTUkr5FWWR5F2hvwG8DJwTKPVlxd1Hfk/vA3Mz7Or9yJ1dJ+YV1PtVG/03Mnvq5XXYu1jOI7yuoOFR0MJ2CXfyloMdFhwshicNyQdQNq4vmfpj05eMbV2vSaRto19CQ8nXKRFxH9LuoYFo7xqR8od1vcr21fFi4AOKx4NNQiS1id1CN8cEWdLWgvYNyKOa/GlmVmTJG3JW1dh8GoFfXCyGCJJm3T6OHqzRY2k3wJrA1NZsIRJRMQXW3dV7c3JYgBqlnSoLat88pyZDY2ke0kDO/wB2CSPhhqY2+qUuTfYrPPMAN7R6ovoJO7gHph6ieGYyq/CzIZqReBvkm5j4fWudm/dJbU3N0MNgKSZwE/7ej4i+nzOzNpHr6Vt3hQR11d9LZ3CNYuB6W8lWDPrEL2TQt6R8gDAyaIPThYD82RJGxyZWcUkbQx8Avg48DBpZQbrg5PFwLhGYdbB8rLxB+TbM8C5pOb4yvah71TusxgASSt08rLMZou6vHDgDcAhEfFgLnvIe3M05qGzA+BEYdbx9gKeBP4s6RRJO+AWg6a4ZmFmixxJSwF7kJqjtidtMXBRRFzZ0gtrY04WZrZIk7Q8qZN7v4iobNOxTuNkYWZmDbnPwszMGnKyMDOzhpwszMysIScLMzNryMnCzMwa+v+sVKwym+zF7AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}