__activelr.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.nn as nn\n",
    "import time\n",
    "import torch.optim as optim\n",
    "\n",
    "from matplotlib.animation import FuncAnimation\n",
    "import ipywidgets as widgets\n",
    "%matplotlib inline\n",
    "\n",
    "# from ConsciousLR import ConsciousLR\n",
    "\n",
    "from IPython import display\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, sample_size=1000, model='2x', fill = 1 ):\n",
    "        super().__init__()\n",
    "        self.model = model\n",
    "        self.fc1 = nn.Linear(1, 1, bias=False)\n",
    "        self.fc1.weight.data.fill_(fill)\n",
    "    def forward(self, x):\n",
    "        out = self.fc1(x)\n",
    "        return out\n",
    "\n",
    "        \n",
    "\n",
    "torch.manual_seed(1)\n",
    "sample_size = 1000\n",
    "x = torch.randn(sample_size, 1)\n",
    "y = x*2\n",
    "y.shape\n",
    "\n",
    "net1 = Net()\n",
    "net2 = Net()\n",
    "\n",
    "class Crit:\n",
    "    def __init__(self, measure=\"MSE\"):\n",
    "        self.measure = measure\n",
    "    def __call__(self, pred, y):\n",
    "        if self.measure==\"MSE\":\n",
    "            return (pred-y).pow(2).mean()\n",
    "        elif self.measure==\"MAE\":\n",
    "            return (pred-y).abs().mean()\n",
    "        else:\n",
    "            return ((pred-y).pow(4)-8*(pred-y).pow(2)).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "# import torch\n",
    "from torch.optim.optimizer import Optimizer, required\n",
    "# import numpy as np\n",
    "class ConsciousLR(Optimizer):\n",
    "    r\"\"\"Implements AdamW algorithm.\n",
    "\n",
    "    The original Adam algorithm was proposed in `Adam: A Method for Stochastic Optimization`_.\n",
    "    The AdamW variant was proposed in `Decoupled Weight Decay Regularization`_.\n",
    "\n",
    "    Arguments:\n",
    "        params (iterable): iterable of parameters to optimize or dicts defining\n",
    "            parameter groups\n",
    "        lr (float, optional): learning rate (default: 1e-3)\n",
    "        betas (Tuple[float, float], optional): coefficients used for computing\n",
    "            running averages of gradient and its square (default: (0.9, 0.999))\n",
    "        eps (float, optional): term added to the denominator to improve\n",
    "            numerical stability (default: 1e-8)\n",
    "        weight_decay (float, optional): weight decay coefficient (default: 1e-2)\n",
    "        amsgrad (boolean, optional): whether to use the AMSGrad variant of this\n",
    "            algorithm from the paper `On the Convergence of Adam and Beyond`_\n",
    "            (default: False)\n",
    "\n",
    "    .. _Adam\\: A Method for Stochastic Optimization:\n",
    "        https://arxiv.org/abs/1412.6980\n",
    "    .. _Decoupled Weight Decay Regularization:\n",
    "        https://arxiv.org/abs/1711.05101\n",
    "    .. _On the Convergence of Adam and Beyond:\n",
    "        https://openreview.net/forum?id=ryQu7f-RZ\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, params, stepSize, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,\n",
    "                 weight_decay=1e-2, amsgrad=False, lrHigh=.05, lrLow=.95):\n",
    "        if not 0.0 <= lr:\n",
    "            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n",
    "        if not 0.0 <= eps:\n",
    "            raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n",
    "        if not 0.0 <= betas[0] < 1.0:\n",
    "            raise ValueError(\"Invalid beta parameter at index 0: {}\".format(betas[0]))\n",
    "        if not 0.0 <= betas[1] < 1.0:\n",
    "            raise ValueError(\"Invalid beta parameter at index 1: {}\".format(betas[1]))\n",
    "        if not 0.0 <= weight_decay:\n",
    "            raise ValueError(\"Invalid weight_decay value: {}\".format(weight_decay))\n",
    "        defaults = dict(lr=lr, betas=betas, eps=eps,\n",
    "                        weight_decay=weight_decay, amsgrad=amsgrad,\n",
    "                        lrHigh=lrHigh, lrLow=lrLow, stepSize=stepSize)\n",
    "        super(ConsciousLR, self).__init__(params, defaults)\n",
    "\n",
    "    def __setstate__(self, state):\n",
    "        super(ConsciousLR, self).__setstate__(state)\n",
    "        for group in self.param_groups:\n",
    "            group.setdefault('amsgrad', False)\n",
    "\n",
    "    @torch.no_grad()\n",
    "    def step(self, closure=None):\n",
    "        \"\"\"Performs a single optimization step.\n",
    "\n",
    "        Arguments:\n",
    "            closure (callable, optional): A closure that reevaluates the model\n",
    "                and returns the loss.\n",
    "        \"\"\"\n",
    "        loss = None\n",
    "        if closure is not None:\n",
    "            with torch.enable_grad():\n",
    "                loss = closure()\n",
    "\n",
    "        for group in self.param_groups:\n",
    "            for p in group['params']:\n",
    "                if p.grad is None:\n",
    "                    continue\n",
    "\n",
    "\n",
    "                # Perform optimization step\n",
    "                grad = p.grad\n",
    "                if grad.is_sparse:\n",
    "                    raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')\n",
    "                amsgrad = group['amsgrad']\n",
    "\n",
    "                state = self.state[p]\n",
    "\n",
    "                # State initialization\n",
    "                if len(state) == 0:\n",
    "                    state['step'] = 0\n",
    "                    # Exponential moving average of gradient values\n",
    "                    state['exp_avg'] = torch.zeros_like(p, memory_format=torch.preserve_format)\n",
    "                    # Exponential moving average of squared gradient values\n",
    "                    state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)\n",
    "                    if amsgrad:\n",
    "                        # Maintains max of all exp. moving avg. of sq. grad. values\n",
    "                        state['max_exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)\n",
    "                    state['gai'] = torch.ones_like(p, memory_format=torch.preserve_format)\n",
    "                    state['cumm'] = torch.zeros_like(p, memory_format=torch.preserve_format)\n",
    "\n",
    "                # Accumulate gradients for the epoch\n",
    "                state['cumm']+=(p.grad)\n",
    "#                 print('step', state['step'], 'cumm', state['cumm'], 'grad', p.grad.item())\n",
    "\n",
    "\n",
    "                # Perform stepweight decay\n",
    "                p.mul_(1 - group['lr'] * state['gai'] * group['weight_decay'])\n",
    "\n",
    "\n",
    "                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n",
    "#                 print('exp_avg',state['exp_avg'])\n",
    "                if amsgrad:\n",
    "                    max_exp_avg_sq = state['max_exp_avg_sq']\n",
    "                beta1, beta2 = group['betas']\n",
    "\n",
    "                state['step'] += 1\n",
    "                bias_correction1 = 1 - beta1 ** state['step']\n",
    "                bias_correction2 = 1 - beta2 ** state['step']\n",
    "\n",
    "                # Decay the first and second moment running average coefficient\n",
    "#                 exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)\n",
    "\n",
    "                state['exp_avg'] = beta1 * (exp_avg) + (1-beta1)*(grad)\n",
    "#                 print('exp_avg',state['exp_avg'])\n",
    "#                 exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)\n",
    "\n",
    "                state['exp_avg_sq'] = beta2 * exp_avg_sq + (1-beta2)*grad.pow(2)\n",
    "                if amsgrad:\n",
    "                    # Maintains the maximum of all 2nd moment running avg. till now\n",
    "                    torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)\n",
    "                    # Use the max. for normalizing running avg. of gradient\n",
    "                    denom = (max_exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps'])\n",
    "#                 else:\n",
    "#                     denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(group['eps'])\n",
    "\n",
    "                # Correction\n",
    "                exp_avgCorr = state['exp_avg']/(1-beta1**state['step'])\n",
    "#                 print('exp_avgCorr',exp_avgCorr)\n",
    "                exp_avg_sqCorr = state['exp_avg_sq']/(1-beta2**state['step'])\n",
    "\n",
    "#                 step_size = group['lr'] / bias_correction1\n",
    "                step_size = group['lr']\n",
    "#                 gai = state['gai']\n",
    "#                 numer = exp_avg.mul(state['gai'])\n",
    "#                 numer = exp_avg * gai\n",
    "#                 print(numer, ' equals ', exp_avg, gai, state['gai'])\n",
    "#                 p.addcdiv_(numer, denom, value=-step_size)\n",
    "                p -= step_size*state['gai']*(exp_avgCorr/(exp_avg_sqCorr.sqrt()+group['eps']))\n",
    "                # SetLR if i>0\n",
    "                if state['step']/group['stepSize'] > 1 and state['step']%group['stepSize']==0:\n",
    "                    tmp2 = state['gradOld'].clone().cpu()##could be eliminated\n",
    "                    tmp3 = state['cumm'].clone().cpu()##could be eliminated\n",
    "                    tmp5 = state['gai'].clone().cpu()##may be the one that needs cloning\n",
    "#                     print(f'old {tmp2}, cumm {tmp3}')\n",
    "#                     print(state['step'],' is ', state['gai'].item())\n",
    "                    state['gai'] = torch.as_tensor(np.where(tmp2*tmp3<=0, tmp5.mul(group['lrLow']), tmp5.add(group['lrHigh'])),dtype=p.dtype , device=p.device)\n",
    "#                     print(state['step'],' is ', state['gai'].item())\n",
    "\n",
    "                # Resetting the accumulated gradients after each epoch\n",
    "                if state['step']%group['stepSize']==0:\n",
    "                    cumm = state['cumm']\n",
    "                    state['gradOld'] = cumm.clone()\n",
    "                    state['cumm'] = torch.zeros_like(p, memory_format=torch.preserve_format)\n",
    "\n",
    "\n",
    "\n",
    "        return loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "out2 = widgets.Output()\n",
    "out0 = widgets.Output()\n",
    "plt.ioff()\n",
    "plt.close('all');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "lossType = widgets.ToggleButtons(\n",
    "    options=['MSE', 'MAE', 'Saddle'],\n",
    "    description='Loss Function',\n",
    "    disabled=False,\n",
    "    button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
    "    tooltips=['Mean Squared Error Loss', 'Mean Absolute Error Loss', 'A Custom Loss Function with a Saddle'],\n",
    "    value='Saddle'\n",
    "#     icons=['check'] * 3\n",
    ")\n",
    "measure = lossType.value\n",
    "\n",
    "outw = widgets.Output()\n",
    "www = widgets.FloatSlider(min=0, max=4, description=\"Initial Weight\")\n",
    "ep = widgets.IntText(description=\"Number of Epochs\", value=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_click(b):\n",
    "    global weight_space\n",
    "    global losses\n",
    "    global crit\n",
    "    global measure\n",
    "    global x1data, x2data, y1data, y2data\n",
    "    global startW\n",
    "    global ax1,ax2, fig\n",
    "    net1.fc1.weight.data.fill_(www.get_interact_value()); net2.fc1.weight.data.fill_(www.get_interact_value())\n",
    "    opt1 = optim.Adam(net1.parameters(),lr=lr)\n",
    "    opt2 = ConsciousLR(net2.parameters(), 1, lr=lr, lrLow=.5, lrHigh=1.5)\n",
    "    for _ in range(ep.value):\n",
    "#         ax1.clear()\n",
    "#         ax2.clear()\n",
    "        time.sleep(1)\n",
    "        w_a = net1.fc1.weight.data.item()\n",
    "        w_c = net2.fc1.weight.data.item()\n",
    "\n",
    "    #     time.sleep(1.)\n",
    "        opt1.zero_grad()\n",
    "        l1 = crit(net1(x), y)\n",
    "        l1.backward()\n",
    "        opt1.step()\n",
    "\n",
    "        opt2.zero_grad()\n",
    "        l2 = crit(net2(x), y)\n",
    "        l2.backward()\n",
    "        opt2.step()\n",
    "\n",
    "\n",
    "\n",
    "        x1data.append(w_a)\n",
    "        x2data.append(w_c)\n",
    "\n",
    "        y1data.append(l1.item())\n",
    "        y2data.append(l2.item())\n",
    "\n",
    "        ax1.plot(x1data, y1data, 'blue', lw=3, alpha=0.4)\n",
    "        ax2.plot(x2data, y2data, 'black', lw=3, alpha=0.4)\n",
    "        \n",
    "        with out2:\n",
    "            display.clear_output(wait=True)\n",
    "            display.display(fig)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lossHandler(change):\n",
    "    global weight_space\n",
    "    global losses\n",
    "    global crit\n",
    "    global measure\n",
    "    global x1data, x2data, y1data, y2data\n",
    "    global startW\n",
    "    global fig, ax1, ax2\n",
    "#     if change['new']\n",
    "    out0.clear_output(wait=True)\n",
    "    out2.clear_output(wait=True)\n",
    "#     time.sleep(1)\n",
    "\n",
    "    \n",
    "    measure = change['new']\n",
    "    \n",
    "#     fig, ax = plt.subplots(1,2)\n",
    "#     for a, loss in zip(ax, losses):\n",
    "#         a.plot(weight_space ,loss, c='red')\n",
    "            \n",
    "    steps = 50\n",
    "    weight_space = torch.linspace(0, 4, steps=steps)\n",
    "    crit = Crit(measure=measure)\n",
    "\n",
    "    loss1 = np.zeros((steps))\n",
    "    loss2 = np.zeros((steps))\n",
    "    for i, w in enumerate(weight_space):\n",
    "        net1.fc1.weight.data.fill_(w)\n",
    "        net2.fc1.weight.data.fill_(w)\n",
    "        with torch.no_grad():\n",
    "            loss1[i] = crit(net1(x), y)\n",
    "            loss2[i] = crit(net2(x), y)\n",
    "    losses = [loss1, loss2]\n",
    "    lr = 1e-2\n",
    "\n",
    "    out2.clear_output(wait=True)\n",
    "    opt1 = optim.Adam(net1.parameters(),lr=lr)\n",
    "    opt2 = ConsciousLR(net2.parameters(), 1, lr=lr, lrLow=.5, lrHigh=1.5)\n",
    "\n",
    "    fig, (ax1, ax2) = plt.subplots(1,2, figsize=(7,4))\n",
    "    ax1.plot(weight_space ,losses[0], c='red')\n",
    "    ax2.plot(weight_space ,losses[0], c='red')\n",
    "\n",
    "    ax1.set_title('Adam')\n",
    "    ax2.set_title(\"ActiveLR\")\n",
    "    \n",
    "    # ax1, ax2 = ax\n",
    "    x1data, x2data, y1data, y2data = [], [], [], []\n",
    "    # ln1, = ax1.plot([], [], 'blue', lw=3, alpha=0.4)\n",
    "    # ln2, = ax2.plot([], [], 'black', lw=3, alpha=0.4)\n",
    "    # line = [ln1, ln2]\n",
    "\n",
    "    ax1.set_xlim(torch.min(weight_space).item(), torch.max(weight_space).item())\n",
    "    ax1.set_ylim((losses[0]).min(), (losses[0]).max())\n",
    "\n",
    "    ax2.set_xlim(torch.min(weight_space).item(), torch.max(weight_space).item())\n",
    "    ax2.set_ylim((losses[0]).min(), (losses[0]).max())\n",
    "\n",
    "\n",
    "    button = widgets.Button(description='Train')\n",
    "    button.on_click(train_click)\n",
    "\n",
    "\n",
    "    out2.clear_output(wait=True)\n",
    "    with out0:\n",
    "        display.display(lossType, button,)\n",
    "    with out2:\n",
    "        display.display(fig)\n",
    "\n",
    "lossType.observe(lossHandler, names='value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAEICAYAAADsh6tqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxeZZUH8N9J0n2hW5pCF7pYahehSmkRRBYpLfsygwIKyACVGTYdBtlUVBgHdRB0dJSyVpQiKiJbQUD2JVCgLG2BLhSatmmbNulG2zTJmT/Oe21gkuZd7r3P87zv7/v55JOlyb0nb+/Juc96RVVBREQUgjLXARAREWWLRYuIiILBokVERMFg0SIiomCwaBERUTBYtIiIKBgsWoETkTtE5FrXcRAVAxH5jYh813Uc1D4WLY+JyFMiUi8iXVzHQhSCXHJGRL4uIs+1/pqqnqeq18QUxzltfH24iKiIbM68LRORyws9Xylh0fKUiAwHcBAABXCc02CIAhBYzvRR1Z4A/hnAd0VkquuAQsGi5a8zALwE4A4AZ0ZfFJHPishrIrJJRP4AoGurf+srIg+KyNrM3eaDIjKk1b8/JSLXisgLmbu8B0Skv4j8XkQ2isgrmcQnClF7OTNURO7N5MU6EfmliIwF8BsAn8/kQkPme//R3S4iC0XkmFbHqRCROhH5XObz/TO51CAib4jIIbkGrKpzAcwHMDHv37rEsGj56wwAv8+8TRORKhHpDOA+AHcC6AfgjwD+qdXPlAG4HcCeAIYB2Argl5847ikATgcwGMAoAC9mfqYfgIUArk7o9yFKWls5Uw7gQQAfABgOu+7vVtWFAM4D8KKq9lTVPm0cbzaAU1t9Pg1Anaq+JiKDATwE4FpY7vwHgD+LSGUuAYvI/gAmAFicy8+VMhYtD4nIF2CF5x5VfRXAEgCnAdgfQCcAN6rqDlX9E4BXop9T1XWq+mdV/UhVNwH4TwAHf+Lwt6vqElXdAGAOgCWq+riqNsGK4GcT/wWJYraLnJkMYA8Al6rqFlXdpqrP7eJQrd0F4DgR6Z75/LTM1wDgawAeVtWHVbVFVR8DMBfAUVkeu05EtsJuGv8XdjNKWWDR8tOZAP6mqnWZz+/KfG0PACv047scfxB9ICLdReQmEflARDYCeAZAn8zdZmR1q4+3tvF5zxh/D6K0tJczQwF8kLkpy4mqLob1PhybKVzHYWfR2hPAyZmuwYZM9+IXAOye5eEHwHLtPwAcArsZpSxUuA6APk5EugH4MoByEanNfLkLgD4AVgEYLCLSqnANg91VAsAlAMYAmKKqtSIyEcDrACS1X4AoZR3kzGoAw0Skoo3Clc0jLqIuwjIACzKFDACWA7hTVc/NN25VbQZwvYicCODfANyY77FKCVta/jkBQDOAcbDB2YkAxgJ4NvNvTQAuygwKnwTr/oj0grWWGkSkHzg+RaWho5xZBeA6EekhIl1F5MDMz60GMCQzVtyeuwEcAeBfsbOVBQC/g7XApolIeea4h7Se+ASgIvP16K291tR1AL4tIl3b+XdqhUXLP2fCxp0+VNXa6A02oeJUACcB+DqAegBfAXBvq5+9EUA3AHWwWVSPpBk4kSMd5cyxAD4F4EMANbC8AYC/w2bu1YpIXRvHhaqugo07HQDgD62+vhzA8QCuBLAW1vK6FB//m/pr2E1k9HZ7O/E/BMvnvFttpUT4EEgiIgoFW1pERBQMFi0iIgoGixYREQWDRYuIiILhZJ3WgM6ddfjee7s4NVHhNm7Eq4sW1alqTlv2JGlAp046fJ99XIdBlJ8tW4B33sGrtk3WLvPKSdEa3qkT5s6d6+LURIWbPRty2mkfdPyN6RleXs6conA99BBwzDGQVjv8tMdN92BTzjuqEPmjrs0lPW4xpyhkOeQUixZRrnwsWs3N9kYUIu+LVksLsH27k1MTFczHogUA9fWuIyDKT10dUJHdaJW72YPr1jk7NVFBfC1avsZF1JG6OmDAgKy+lUWLKFe+Xru+xkXUkXXrAihavCukUPl67foaF1FH6uqA/v2z+lYWLaJc+Xrt+hoXUUeC6B5kglGIVP29dn2Ni6gjLFpECdmyxc+ZryLMKQpTS0sAY1plZRw0pjD5et1WVPgbG9GubNhghcvrolVRwbtCCpOv1y1zikIVXbdeT8RgglGofL1umVMUqui6ZUuLKAG+XrfMKQoVixZRgny9bplTFKpgihYHjSlE69bZTD3fVFQADQ3cjJrCE9UC74vW5s3Atm1OTk+UtxxW7qeqosLWkHHTXApNXR3QqRPQs2dW35510RKRoSLypIgsFJH5InJx5uv9ROQxEVmUed+3w4NFu/mytUWhibFoJZJT7CKk0EQLi7PswcilpdUE4BJVHQtgfwDni8g4AJcDeEJVRwN4IvP5rjHBKFQ5rNzPAnOKKMecyrpoqeoqVX0t8/EmAAsBDAZwPIBZmW+bBeCEDg/GBKNQxVi0mFNESK5otSYiwwF8FkA1gCpVXQVYEgIY2M7PzBCRuSIyt37TJvsiuwcpNDlsN5OLQnNq/caNO+MjCkmOOZVz0RKRngD+DOCbqrox259T1ZmqOklVJ/WtrLQv8q6QQhJtlhvzRIw4cqrfwExdY05RaHLMqZyKloh0giXX71X13syXV4vI7pl/3x3Amg4PxK4MCtHmzUBjY6wtrdhyqqwM6NaNOUVhaW4G1q9PpqUlIgLgVgALVfVnrf7pfgBnZj4+E8BfszgYsNtuTDAKS46LIDsSa05FcTGnKCQNDTltlgsAFTkc/kAApwN4S0TmZb52JYDrANwjImcD+BDAyVkdjQlGoYm5aIE5RaUuj5zKumip6nMA2ptI/6WszxgZMICDxhSWHFfud4Q5RSUvj5xy9xDI/v15V0hhyfERCqljTlFo8sgpd0WLXRkUmvi7B+PFnKLQ5JFTLFpE2aqrs1l6ffq4jqRtAwbYwPaOHa4jIcpOcEXro4+ArVudhUCUk2g9SZm7tNmlKPHXr3cbB1G26uqALl2AHj2y/hG3RQvgwDGFI6HdMGLDnKLQRDmVw+N+3E7EANhFSOHw9bEkEeYUhSaPnHLf0mKCUSji3eE9fswpCk0eOcWiRZQtFi2ieLFoESUk2izX56LF7kEKTVBFq18/e89BYwrBpk1AU5PfRatbN5uFxZyiEDQ3A/X1ARWtigpb78K7QgqB77thRLgrBoWivt56MIKZiAFwgTGFw/fdMCLMKQpFnjnFokWUDRYtonixaBEliEWLKF7BFi0OGlMIYn4sSWKYUxSKPHPKbdHioDGFoq4OKC+3J277rH9/YMMGbppL/stzcpP7ltbWrbZxLpHPou1mctgjzQnuP0ihqKsDunYFunfP6cfcFy2ArS3yn+8LiyPMKQpFlFM53giyaBFlg0WLKF555pQfRYtdGeQ73x9LEmFOUSjyzCk/itaaNU7DIOrQmjVhFS3mFPkuz5xyW7QGDbL3q1c7DYNol5qarCsjul59VllpYwTMKfLd6tV55ZTborXbbkDnzkBtrdMwiHZp7VrbI62qynUkHevUyWY5MqfIZ1u2AJs355VTbouWiFVa3hWSz6LrM4SWFsCcIv8VkFNuixZglZZ3heSz6PoMoaUFMKfIfwXklB9Fi3eF5LPo+gypaDGnyGcF5JT7osWuDPJdqN2Dqq4jIWpb8N2Da9bYUyyJfFRba1vN9OzpOpLsVFXZ1mibN7uOhKhtUfdgZWXOP+q+aA0aBLS0cDEk+SvPqbnOcCkJ+W71aluj1alTzj/qvmhFfZocOCZf1daGM54FMKfIfwXkVNZFS0RuE5E1IvJ2q699X0RWiMi8zNtROUfAu0LyXUItLeYUlawCciqXltYdAKa38fUbVHVi5u3hnCPgXSH5LrmW1h1gTlEpSqOlparPAFif11l2hXeF5LMdO2y8NYGWVmI5VVkJlJUxp8hPqqm1tNpzgYi8menq6NveN4nIDBGZKyJz165du/MfevWyB4HxrpB8FG08m+6YVmE5VV5ug9zMKfLR5s02uzXpllY7fg1gFICJAFYBuL69b1TVmao6SVUnVbae5sitnMhn6a/RKjynAOYU+avAnCqoaKnqalVtVtUWADcDmJzXgbiCn3yV8m4YzCkqegXmVEFFS0R2b/XpiQDebu97d2nQIHZlkJ+i6zKllhZziopegTlVke03ishsAIcAGCAiNQCuBnCIiEwEoACWAfhGXlFUVQEvvpjXjxIlKsGWVuI5FW3lJBJPwERxKDCnsi5aqnpqG1++Na+zflJVlT2zqKkJqMg6JKLk1dba9k3du8d+6MRzats2YONGe24dkS9qa+1GKs8ngbvfEQOwZqKqPR2WyCehbeEU4VIS8tXq1bYsI88Gih9Fi4shyVehbeEUYU6RrwrMKT+KFu8KyVdsaRHFq8Cc8qNo8a6QfMWWFlG82NIiSkhjI1BfH2ZLq39/2xmDOUU+KXALJ8CXohXNzmKCkU/cbOEUj/JyG+xmTpFPNm2yWa3Bt7QALoYk/6S8sDh2zCnyTQw55U/R4rYz5JuUt3CKHXOKfBNDTvlTtHhXSL5hS4soXmxpESWoWFpaqq4jITJF1dKqqrIdMXbscB0JkamtBXr3tue9haiqymZANjS4joTI1NbaA0r798/7EP4Urai52PphdkQuhbqwOMKlJOSb1auBgQNtdmue/ClaXAxJvgl1YXGEOUW+iSGn/ClavCsk37ClRRSvGHLKn6IVVV8mGPli9eriaGkxp8gXMeSUf0WLXRnkg23bbAJDyC2tfv3s8Q/MKfKBql2LRdPS6tHDtnPiXSH5IOQtnCJlZTbozZwiH2zYYLNZi6alBXAxJPkj9IXFEeYU+SKmnPKraHGBMfki9IXFEeYU+SKmnPKraPGukHzBlhZRvIq2pcUEIx9Ed4UDB7qNo1BRS6ulxXUkVOqKsqU1eLA9dG/rVteRUKlbsQIYMADo0sV1JIUZPBhoarIt0ohcWrEC6Ny5oC2cAN+K1pAh9r6mxm0cRMuX77weQxb9DsuXu42DKMopkYIOw6JF1JaamuIqWswpci2mnGLRImoLixZRvFi0iBKydSuwbh0wdKjrSAo3cCDQqRNzitxqabExrRhyyq+i1b27bT3DBCOXVqyw98XQ0iors8kYzClyqa7OdsMoupYWYL8UE4xciq6/YihaAHOK3Isxp/wsWpzpRC5F118xFS3mFLkUY075WbR4V0guRdff4MFu44hLlFOqriOhUuWipSUit4nIGhF5u9XX+onIYyKyKPO+b8ERDRkCrF1rj4YgcqGmBujb1548kKBUc2r7dptcQuRCTY09JieGHWZyaWndAWD6J752OYAnVHU0gCcynxcmqsQrVxZ8KKK8pDfd/Q6kmVPswSBXamqs56Ks8M69rI+gqs8AWP+JLx8PYFbm41kATig4omhKJBOMXKmpSWW6O3OKSkaMOVVo2atS1VUAkHlfeNuPd4XkmtuFxcwpKj4x5lRqEzFEZIaIzBWRuWvXrm3/G6PBb852Ihe2b7enFgcwczDrnKqqAsrLmVPkhqpXRWu1iOwOAJn3a9r7RlWdqaqTVHVSZWVl+0fs1QvYbTfeFZIb7hcWx59T5eXAHnswp8iNdetsYp0nRet+AGdmPj4TwF8LPJ7htHdyxf3CYuYUFZeYcyqXKe+zAbwIYIyI1IjI2QCuAzBVRBYBmJr5vHBMMHIlxaLFnKKSEHNOVWT7jap6ajv/9KVYImltyBDgjTdiPyxRh1IsWqnn1EMP2fhCgc8zIsqJq5ZWqoYOtUczNza6joRKTU2Njan26uU6kngNHQp89BHQ0OA6Eio1NTU2rjpoUCyH87NoDRlid4RcYExpK5YnFn8Sn2BMrixfbhOBystjOZy/RQtgHzylr1ge/vhJzClyJeacYtEiao1FiyheLFpECWlstLHUYixagwbZvm/MKUpTzAuLAV+LVu/eQM+eTDBK16pVlmTFWLQ6dbLCxZyiNDU02ASgoi9aIlxXQulzv7A4WcwpSlsCOeVn0QJsii5nOlGaousthR3enWBOUdoSyCl/ixbvCiltpdDSWr6cTzCm9JRUS2vIEBtj2LHDdSRUKmpqbCy1d2/XkSRjyBBgyxZg40bXkVCpqKmxCUAxLSwGfC9aqkBtretIqFREs5yKdZsjzsqltNXUWMHq1Cm2Q/pdtAAmGKWnWNdoRZhTlLYEcopFiyjCokUUr5IsWpztRGloarIx1GIuWnvsYV2fzClKg2oie3n6W7T69gW6d2eCUTpWrgRaWoq7aHXuDFRVMacoHRs2AJs3l1DREgGGDwfef991JFQKoutsxAi3cSSNOUVpSSin/C1aADByJLB0qesoqBRE19nIkW7jSBpzitKSUE75XbRGjbJfnIshKWlLl9p6kj33dB1JskaNsu5BPmCVkhYVrVGjYj2s30Vr5EhbDLlmjetIqNgtWQIMGxbrehIvjRxpY3cffOA6Eip2S5YA/frZk8Bj5H/RAtidQclburT4uwYB5hSlJ6Gc8rtoRc1KJhglbenS2LsxvMScorQklFN+F63hw+09E4yStGkTsHZtabS0dt8d6NKFOUXJamqyLuiSa2l16wYMHmx9o0RJSWjA2EtlZfaHhDlFSVq+3ApXybW0AE7RpeSVynT3CHOKkpZgTrFoEZVq0eJSEkpKSRetUaOAFSuArVtdR0LFaskS2zasb1/XkaRj1Cgbx6urcx0JFaslS2z5SALbovlftKJKvWyZ0zCoiJXKdPcIp71T0pYutYl05eWxHzqcosUEo6SwaBHFK8Gc8r9oRbNPONuJktDcbK34Upg5GIk2MGVOUVKWLEksp/wvWpWVQI8evCukZNTUADt2lFZLq3t3W6/FnKIk1NcDDQ0l3NIS4QxCSk6pzRyMMKcoKQnnVEUcBxGRZQA2AWgG0KSqk+I47j+MGgUsWhTrIYkAeLuwOJWcevLJWA9JBCDxnIqzpXWoqk6MPbkAriuh5CxZAlRU+PrE4mRzqqYG2L499kNTiYvGShN6oKr/3YOAVeytW4HaWteRULGJpuZWxNLpEI5Ro+wmkEtJKG5LlwIDBwK9eiVy+LiKlgL4m4i8KiIz2voGEZkhInNFZO7atWtzOzqn6FJS/J3uzpyiMCWcU3EVrQNV9XMAjgRwvoh88ZPfoKozVXWSqk6qrKzM7ejRC8ApuhS3JUt8LVrMKQpTwjkVS9FS1ZWZ92sA/AXA5DiO+w/Dh9ssQt4VUpwaGoD1672bhAGkkFNVVTb1nTlFcdqxA/jww0RzquCiJSI9RKRX9DGAIwC8XehxP6ZzZ2DoUCYYxev99+29Zy2tVHKKS0koCR9+CLS0JJpTcYw+VwH4i4hEx7tLVR+J4bgfx2cAUdyi68mzogXmFIUqhZwquGip6lIA+8QQy66NGgU89FDip6ES4unC4lRz6vHHbRahFUiiwqSw7jGMKe+A/WGprQU++sh1JFQsli4FBgwAevd2HYkbI0daPq1Z4zoSKhZLlwJdutg2YQkJq2gB7M6g+Pg7czAd0e++eLHbOKh4LFlii4rLkist4RStT3/a3i9c6DYOKh4LFuy8rkoRc4rilkJOhVW0ysrsRSEqVEMDsHIlMH6860jcGT7cpr0zpygOjY22R2zCORVO0era1Qb35s93HQkVg+gPdSkXrbIyYOxY5hTF47337Pl0LFqtjB/PBKN4RNdRKRctgDlF8Ukpp8IqWuPGWfOzsdF1JBS6+fOta2zYMNeRuDVuHLBihXWXEhVi/nxrve+1V6KnCatojR8PNDVZM5SoEPPn2x/sBGc5BSG6K+a4FhVq/nzgU5+yoZwEhZWxTDCKy4IF7BoEmFMUn5RyKqyiNWaM3RmzD54KEc0cHDfOdSTu7bmndZMyp6gQ27fb0E0KORVW0eIMQooDJ2HsxBmEFIeUZg4CoRUtgLOdqHAsWh/HnKJCpZhTYRatRYusOUqUj/nzgR49OHMwMn68dZdyBiHlK5o5OGZM4qcKr2iNG2fN0EWLXEdCoVqwwLrESn3mYCQah+BkDMrXggU2c7BLl8RPFV7WRs1PdmdQvubPZ9dga8wpKlSKORVe0eIMQipEfT2wahWLVmucQUiF2L7dnhTAotUOziCkQkTXDae778QZhFSId9+1IZuUciq8ogVwthPljzMH28aconylnFPhFq3FizmDkHK3YAFnDrZl/HjrNq2vdx0JhWbBAqC8PJWZg0DIRau5mXsQUu6452DbuJ0T5SvaczCFmYNAqEUr6jtldwblKipa9HHMKcpXyjkVZtHiDELKx/r1QG0tx7PawhmElI9t21KdOQiEWrS6drXmKLsyKBd8WnH7ysrsbpk5Rbl47z2gpYVFKysTJgBvvOE6CgrJm2/aexattkU5peo6EgqFg5wKt2jttx+wZAmwbp3rSCgU1dXAwIGcOdie/fYD1q4FPvjAdSQUiupqm43LMa0sTJ5s7195xW0cFI6XX7brRsR1JH6Kcurll93GQeF4+WVg331tyntKwi1akybZH5/qateRUAgaGoB33gGmTHEdib/23tumLTOnKBvbtwPz5qWeU+EWrd69rUnKBKNsRC1yFq32de4MfO5zzCnKzrx5QGMji1ZOJk+25ikHjqkjUZfXfvu5jcN3kycDr70G7NjhOhLyXZRTUbdySsIuWlOm2ESMpUtdR0K+q6629X19+riOxG9TpgBbtwJvv+06EvJddTWw++7AkCGpnjaWoiUi00XkXRFZLCKXx3HMrETNUnZn0K6o2jUSUNcgc4q8F+VUyhObCi5aIlIO4FcAjgQwDsCpIpLO/McJE4Bu3TjbiXbtww+BNWuCKVpOc2rECGDAAOYU7dr69bYThoOcqojhGJMBLFbVpQAgIncDOB5A8kvrKypsumWp3xXW1QHPPmtva9fu/HpFhQ2sH3ywFfhS3SQ2uj5S7nsvgLucErHXqdRzav164LnnLKdqa3d+vbwcmDgR+OIXgX32SXWqt1ccjWcB8RStwQCWt/q8BsD/K78iMgPADAAYFufizilTgF/+0maxdO4c33F9t3Ej8OtfA7/97c6td7p2BfbYY+f3fPQRcMcd9nGfPsC0acCll1qhLyXV1TaVe++9XUeSLfc5NWeOXWO9e8d3XN9t2QLcfDNw++3AW29Zt3KXLpZTURfYtm3ArFn2ce/ewOGHA5dcAhxwgLu4Xaiuttdk0qTUTx3HrXdbHZr/bzqfqs5U1UmqOqmysjKG02ZMmWLrBaLtRIpdXR3w3e/aBqeXX247PPzXfwHPPw9s2GC7hERvq1YBy5YBd94J/PM/A48+ahfZtGnAM8+UzqzLl1+2Fmc4NzXuc0oVmDs3vmP6rKEBuPZay6lvfQvo2RO45hrLkYYGm+gV5dSKFcDy5cBddwGnngo8/TRw4IHAoYcCjz1WWjk1bpybmxpVLegNwOcBPNrq8ysAXLGrn9l33301NsuWqQKqv/xlfMf0UUuL6i23qPbsab/vSSepzp2b2zE2bFC97jrVgQPtGMcdp1pbm0y8vmhsVO3WTfXii2M9LIC5WmDutPfmPKfWrbPr40c/iu+YvrrrLtU+fez3PeYY1eefz+3nN29WveEG1cGD7RhTp6ouX55MrL5oaVHt31/1rLNiP3Q2eRVHS+sVAKNFZISIdAZwCoD7YzhudoYNA6qqirsPfs0a4IQTgHPOsXVG8+cDf/5z7t18vXsDl11mra+f/MRaXp/5DHB/ev9dqXv7bZvCHcgkjAy3OdWvHzB6dHHn1Pr11lI67TRg7Fjg9deBBx7IvZuvRw/gm9+0VtgvfmE9Hp/5DPCHPyQTtw+WLrWlRo5yquCipapNAC4A8CiAhQDuUdX0HsojYi9esc52euQRm0Tx6KPAz34GPP544ZtTdutmY1uvvgoMHgwcf7wVxG3b4onZJ9F1EVDRcp5TgL1e1dXF2d319NM2vvmnP1m34DPP2OSKQnTpAlx4oe0SMWYMcMopwFe/CmzeHE/MPnGcU7FMJ1PVh1V1L1Udpar/GccxczJ5MvDuu0B9feqnTtRNNwFHH20L+ObOtf72OGcAjh9vf5guvxy49VbgiCPsDrSYVFfbFO4RI1xHkhMvcqq2FqipSf3Uifr974GpU23c6qWXgKuuslm2cRk92mYd/vCHwN13A4ccAqxeHd/xfVBdbTe+EyY4OX1xzIGOKn6x7PiuCnzve8B55wHTp1uXQ1IXSOfONpHj7rvtYvzCF2xdU7GorubO7vkotkXGqtYl/rWv2cSJl15KbhZtRYVNlvrrX21m7wEHAIsWJXMuF6qr7bWLs9jnoDiK1n77WQvk2WddR1K4HTuAs8+22Utnn20Xfs+eyZ/3K1+xLsiVK4H99y+OB2zW1QELF9rvQ7nZZx9bQlEMOdXcDFx8sY3nfuUr1uWexnZexxwDPPmkLR044IDiuAHYssX2pnSYU8VRtHbbzV7ERx91HUlhmpuBM86wdSJXX21rRtK8mznkEOvaKCuzKbxvvZXeuZMQTUGeNs11JOHp0sWuh9BzStV6LP7nf4B//3ebqt6lS3rnnzIFeOGFnWu6Qh97f+opWxPrMKeKo2gB1o02d+7Hd4QISUsL8I1vWDfdddcB3/++my6tCRPs7rpbN+v7f++99GOIy5w5QP/+pbeYOi7Tp9tY8fvvu44kP6pWqG65BbjySuD6693sCjN6tE32qKy01zTkNaVz5gDduwMHHeQshOIpWkceaRfp3/7mOpLcRcl1663Ad75j3RgujRgBPPGEFdLDDw/z8estLdZKmDatdLfaKdSRR9r7Rx5xG0e+rr4auPFG4KKLbJagS4MHW0517243g+++6zaefD3yCHDYYem2Vj+heIrW5z5ndzJz5riOJHdXXw38/Oe23uOHP3Qdjfn0p+0GYNMm4Etfst01QvL667a+bfp015GEa/Rou4EJMad++tOd48I33ODHRJzoZhAI82Zw0SJbj+Y4p4qnaJWV2V31o4/aXXYobr55Z3L97Gd+JFdk4kS7s6qtBY491gZhQxG1DjielT8Ra239/e+2VVooZs8Gvv1tm3Rx001+bRQ9ZoyNtW7eDBx1lG0TFYoop6IWuCMe/W/G4Ow1kwQAAA44SURBVMgjbcbYq6+6jiQ7jz0G/Ou/2p3Lb37jV8GKTJli42yvv27ThZubXUeUnTlzbJ/FgQNdRxK2I4+0m5XnnnMdSXZeeAE46yzbhX3WLD+7hvfeG7j3XhsvPvnkcJ4SPWcOsNdewMiRTsMorqJ1xBH2hz+EPvj5820T23HjbMsXR2sesnLMMdbFct997sfbslFfD7z4ovNujKJw6KG2li+EnFqyxHZ3GTrUioLDcZcOHXooMHOm7XBz/vn+7zyydavNHPQgp4qraA0YYGu2fO+DX73adrro3h148MEwHv9w4YWWXNdfb10uPnv8cesidtyNURR69LBWi+85VV9vOdXSAjz0kM0a9d1ZZwFXXGFDBP/9366j2bVnnrHC5UFOFVfRAuxOoLra3+2IGhuBf/onmyRw//224W8IRGwm1lFHWfF66inXEbVvzhxbPBrOQx/9Nn269QwsX97x97rQ1GR7/S1dai2svfZyHVH2rr3Wuggvu8xuYH01Z44tNj/4YNeRFGHROvJIu9t67DHXkbTtootsW6bbb7dWYUgqKmyQe/RoSzQfZz+pWlfWEUf43eUaEt+nvl95pc10/d//9eKPak7KyuxBrRMn2ga7vk6Ff+QRW2zerZvrSIqwaO23nz1awcfujJtusrdoO5kQ9e5tW0s1NgInnmhPR/bJm2/a9HwPujGKxtix1iPgY07Nnm3T2//t3+xJBSHq3h34y19s7PD44+1hrj55/30rpp7kVPEVrfJyu8t+5BG/pr4//7yNC02fDvxn+pt2x2qvvWw7nHnzgHPP9WsQmVPd4ydi1+3jj/s10+31122pyEEH2UShkO25pz0qZfFi4PTT/frbFeWUB5MwgGIsWgBw3HE22cGXcZcVK2wca8897Y+9j9Nwc3X00dYff9ddNjnDB6oWz6RJ9jgXis9xx9lC84cech2Jqauzln7//sAf/2itlNAdfLCNGz/wgG3j5ou77rIb1dGjXUcCoFiL1gknAH372qwc17Zvt4K1ZYtNGe/b13VE8bniCpu2f9lldhfu2iuvWPdgqN1EPps2DdhjDz9yqqnJutdra61brarKdUTxOf98m1V4zTXWDe/awoW2Ru+cc7xZR1qcRatbN2ti33uv3ZG5dOGFNptx1ix76GIxEbEJJWPH2uytZcvcxjNzpo0PnHqq2ziKUUUF8C//YuNarp+3dvnltkvHb35jrepiImITSiZNsr9h77zjNp6bbwY6dQLOPNNtHK0UZ9ECbKylsRH47W/dxTBzpv2nX3klcNJJ7uJIUs+edrfb1GS/49atbuLYtMl27jjllDDWvYXo7LPt/W23uYth9mzrjr7gAuDrX3cXR5K6drUb7q5drddo40Y3cWzfbn8/jz/eq51lirdoTZhgz9i6+WY3EwVefNESa9o0fzbBTcro0fYY83nzgBkz3Lzes2dbF+y556Z/7lIxfLjtUH7rrW6283rjDSucX/iC7dNZzIYOtbG6xYutleNiYsZf/gKsW+ddThVv0QLsD+g776S/b1o08WLo0OKZeNGRo48GfvAD4He/czOTa+ZM4DOf2fmYeErGjBlATU36a7bWrt05Vv3HP1qXVbE7+GBrVd53n5sb35kz7Ubl8MPTP/cuFHfR+vKXrasozcHjjz6y5vSmTTaQ2q9feud27aqrrFhfeinw8MPpnfe112yT5HPP9WawuGgde6x1FaWZU9EuMtHEi0GD0ju3axddZN2gP/gBcM896Z130SLgySdtAoZPu+Sj2ItWjx7AaafZnVl9ffLnU7WZP6+9Zt1VEyYkf06flJXZhJN99rHJEAsWpHPem2+2/v+vfS2d85Wyzp3tj+iDDwIrVyZ/PlVbOPzsszaWVmpbc4nYhJMDD7TXPa0nWNxyi/UQnXVWOufLQXEXLcC6M7ZtA+68M/lzXXON3Q39+Me2M3op6tHDWpjdutld+bp1yZ5vyxYbTzv55OJaTuCzc86xMa00JmT8/Oc7n+hdqrNCu3SxiRmVlbZeLumbhcZG21rqmGNsmYNvVDX1t3333VdTNWWK6pAhqps2JXeO2bNVAdUzzlBtaUnuPKF48UXVLl1Uv/hF1a1bkzvPD39or/vzzyd3jjYAmKsOcqe9t9RzaupU1QEDVNetS+4cDzygWlameuKJqs3NyZ0nFG+8odqjh+qkScn+LbvxRsupOXOSO0c7ssmr0kiw556zX/Xb307m+HPmqFZUqB50kOq2bcmcI0R33WWv+wknqO7YEf/xly5V7dpV9eST4z92B0q+aM2bZwXlvPOSOf4zz9j/7b77qm7enMw5QnT//fa6T52azN+alStVe/VSnTbNyc03i1ZrX/+6FZYFC+I97vPPq3bvrjpxompDQ7zHLga/+IVdZmedFX8SHHus3XkuXx7vcbNQ8kVLVfWii1RFVF95Jd7jvv66au/eqmPGqK5ZE++xi8Htt1tOnXyyalNTvMf+6ldVO3dWfe+9eI+bJRat1lavVu3TR/Www+L74/nmm3bMT31KtbY2nmMWo6uvtkvtkkvie+3vv9+O+ZOfxHO8HLFoqd2kVVWp7rdffN13772nOnCg6tChqh9+GM8xi9H119v1P2NGfDn11FN2zO98J57j5YFF65N+9Sv7le++u/BjvfGG6qBBqnvsofr++4Ufr5i1tKhecIG99ldeWXiSffSR6ogRqmPHqm7fHk+MOWLRyrjzTvt/nTmz8GO9847qnnvaWNnChYUfr9hdeaW99hdeWHiLq7FRdfx4e/23bIklvHywaH1SU5PqZz9rhaaQrry//c36fQcPVp0/P774illzs+q559old/rphRWb737XjvPEE/HFlyMWrYyWFhvL7devsK68Z5+1Y1RWqr76anzxFbOWFtVvfcty4aST7GYuXz/9qR3nvvviiy8PLFpteekl1fJy69JYuzb3n7/tNhsb23tvJ2MpQWtpUb3mGrvsDjtMtb4+92P8/Of286edFn98OWDRauWtt2wcZMIEG8jP1T332EzTvfZSXbIk/viK3Q032Nji5z+f343Drbfa5I5jj3U+8znxogXg+wBWAJiXeTsqm59zmmCqqn/9qyXJ2LHZF56NG3fe1UydqrphQ7IxFrPf/tYK/7hxdhORjZYW1e99T/8xGzHJafRZSKpoBZtTjz1mk2JGjFBdvDi7n9myRfWqq+z/9MADVevqko2xmP3pTzbbctQo1aefzv7nohbW1KnJTqPPUlpF6z9y/TnnCaZqg469eqkOG6b67rvtf19zs+odd9j4FWBTfBsb04uzWD3xxM7X9Iwzdn2H3tysev75+o9ZiElMn89RwkUrzJyqrrYuvkGDbMy3PS0tNq48dOjO/3/HNyFF4YUXdr6mX/6y6rJl7X9vS4vq5ZfrP2YherJUh0WrI6++an3onTurHnWUDSavWmV/QB9+WPVHP7KFfIDq5MnZtwooOxs3ql52mb3+PXvagPKsWfYHb8sWK2wXXmg3FnHPPiwQi1Y75s+3sd6KCrt7/9WvVGtqbPbuo4+q/vjH1qoCbJlILq0C6tiWLTZbt2tXezvvPBvSeO01+7enn7Yeo5Ej7f/gG9+If9p8AdIqWssAvAngNgB9d/G9MwDMBTB32LBhqbwAWVmyxP4TR4ywl+OTb6NH2x9SrshPzqJFNpDctev/f/27dlU97ji7M/ekYKkmXrTCzqnly1UvvdRyp62cGjFC9aabvPpjWXQ++ED11FNtDeknX//oJn3WLK9ySjW7vBL7vvaJyOMA2tpW+SoALwGoA6AArgGwu6r+yy4PaMfcBODdjr7PIwNgv2dIQos5tHjHqGqvfH6QOQUgvP9vILyYQ4sXyCKvOixa2RKR4QAeVNUOtzYXkbmqGsxzskOLFwgvZsbb5jmGgznljdBiDi1eILuYC9rlXUR2b/XpiQDeLuR4RKWOOUW0axUF/vxPRGQirCtjGYBvFBwRUWljThHtQkFFS1VPz/NHZxZyXgdCixcIL2bGC+aU50KLObR4gSxijm1Mi4iIKGnF/+RiIiIqGixaREQUjFSLlohMF5F3RWSxiFye5rnzISK3icgaEQliBpeIDBWRJ0VkoYjMF5GLXcfUERHpKiIvi8gbmZh/4DqmbIhIuYi8LiIPOo6DOZUg5lR6ss2p1IqWiJQD+BWAIwGMA3CqiIxL6/x5ugPAdNdB5KAJwCWqOhbA/gDOD+A13g7gMFXdB8BEANNFZH/HMWXjYgALXQbAnEoFcyo9WeVUmi2tyQAWq+pSVW0EcDeA41M8f85U9RkA613HkS1VXaWqr2U+3gS7AAa7jWrXMru3bM582inz5vXsIBEZAuBoALc4DoU5lTDmVDpyyak0i9ZgAMtbfV4Dz//zQ5bZTeGzAKrdRtKxTLfAPABrADymqr7HfCOAbwNocRwHcypFzKlEZZ1TaRYtaeNrXlf/UIlITwB/BvBNVd3oOp6OqGqzqk4EMATAZBHpcNsiV0TkGABrVPVV17GAOZUa5lRycs2pNItWDYChrT4fAmBliucvCSLSCZZcv1fVe13HkwtVbQDwFPwe8zgQwHEisgzWHXeYiPzOUSzMqRQwpxKXU06lWbReATBaREaISGcApwC4P8XzFz0REQC3Alioqj9zHU82RKRSRPpkPu4G4HAA77iNqn2qeoWqDlHV4bBr+O+q+jVH4TCnEsacSl6uOZVa0VLVJgAXAHgUNph5j6rOT+v8+RCR2QBeBDBGRGpE5GzXMXXgQACnw+5U5mXejnIdVAd2B/CkiLwJ+yP8mKo6nUYeCuZUKphTnuE2TkREFAzuiEFERMFg0SIiomCwaBERUTBYtIiIKBgsWkREFAwWLSIiCgaLFhERBeP/AC5M1b6g8sXDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "# measure='MSE'\n",
    "steps = 50\n",
    "weight_space = torch.linspace(0, 4, steps=steps)\n",
    "crit = Crit(measure=measure)\n",
    "\n",
    "pause=False\n",
    "\n",
    "loss1 = np.zeros((steps))\n",
    "loss2 = np.zeros((steps))\n",
    "for i, w in enumerate(weight_space):\n",
    "    net1.fc1.weight.data.fill_(w)\n",
    "    net2.fc1.weight.data.fill_(w)\n",
    "    with torch.no_grad():\n",
    "        loss1[i] = crit(net1(x), y)\n",
    "        loss2[i] = crit(net2(x), y)\n",
    "losses = [loss1, loss2]\n",
    "lr = 1e-2\n",
    "\n",
    "out2.clear_output(wait=True)\n",
    "opt1 = optim.Adam(net1.parameters(),lr=lr)\n",
    "opt2 = ConsciousLR(net2.parameters(), 1, lr=lr, lrLow=.5, lrHigh=1.5)\n",
    "\n",
    "fig, (ax1, ax2) = plt.subplots(1,2, figsize=(7,4))\n",
    "ax1.plot(weight_space ,losses[0], c='red')\n",
    "ax2.plot(weight_space ,losses[0], c='red')\n",
    "\n",
    "ax1.set_title('Adam')\n",
    "ax2.set_title(\"ActiveLR\")\n",
    "\n",
    "x1data, x2data, y1data, y2data = [], [], [], []\n",
    "\n",
    "ax1.set_xlim(torch.min(weight_space).item(), torch.max(weight_space).item())\n",
    "ax1.set_ylim((losses[0]).min(), (losses[0]).max())\n",
    "\n",
    "ax2.set_xlim(torch.min(weight_space).item(), torch.max(weight_space).item())\n",
    "ax2.set_ylim((losses[0]).min(), (losses[0]).max())\n",
    "\n",
    "\n",
    "   \n",
    "button = widgets.Button(description='Train')\n",
    "button.on_click(train_click)\n",
    "\n",
    "\n",
    "out2.clear_output(wait=True)\n",
    "with out0:\n",
    "    display.display(lossType, button)\n",
    "with out2:\n",
    "    display.display(fig)\n",
    "    \n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weightHandler(change):\n",
    "    global startW\n",
    "    startW=change['new']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "www.observe(weightHandler, names='value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "outw.clear_output(wait=True)\n",
    "with outw:\n",
    "    display.display(www, ep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15efd20dffb9443bb884080b68631e64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "outw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c699a9046c3d448b8935f1c844e2f164",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output(outputs=({'output_type': 'display_data', 'data': {'text/plain': \"ToggleButtons(description='Loss Functi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "out0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "239926cb7b8f45f381633fadb9ad42c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output(outputs=({'output_type': 'display_data', 'data': {'text/plain': '<Figure size 504x288 with 2 Axes>', 'i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "out2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}