Capsule Network - FullDataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "df = pd.read_csv('sample/Data_Entry_2017.csv')\n",
    "\n",
    "diseases = ['Cardiomegaly','Emphysema','Effusion','Hernia','Nodule','Pneumothorax','Atelectasis','Pleural_Thickening','Mass','Edema','Consolidation','Infiltration','Fibrosis','Pneumonia']\n",
    "#Number diseases\n",
    "for disease in diseases :\n",
    "    df[disease] = df['Finding Labels'].apply(lambda x: 1 if disease in x else 0)\n",
    "    \n",
    "#remove Y after age\n",
    "df['Age']=df['Patient Age'].apply(lambda x: x[:-1]).astype(int)\n",
    "df['Age Type']=df['Patient Age'].apply(lambda x: x[-1:])\n",
    "df.loc[df['Age Type']=='M',['Age']] = df[df['Age Type']=='M']['Age'].apply(lambda x: round(x/12.)).astype(int)\n",
    "df.loc[df['Age Type']=='D',['Age']] = df[df['Age Type']=='D']['Age'].apply(lambda x: round(x/365.)).astype(int)\n",
    "# remove outliers\n",
    "df = df.drop(df['Age'].sort_values(ascending=False).head(16).index)\n",
    "df['Age'] = df['Age']/df['Age'].max()\n",
    "\n",
    "#one hot data\n",
    "# df = df.drop(df.index[4242])\n",
    "df = df.join(pd.get_dummies(df['Patient Gender']))\n",
    "df = df.join(pd.get_dummies(df['View Position']))\n",
    "\n",
    "#random samples\n",
    "df = shuffle(df)\n",
    "\n",
    "#get other data\n",
    "data = df[['Age', 'F', 'M', 'AP', 'PA']]\n",
    "data = np.array(data)\n",
    "\n",
    "labels = df[diseases].as_matrix()\n",
    "files_list = ('sample/images/' + df['Image Index']).tolist()\n",
    "\n",
    "# #test to perfect\n",
    "# labelB = df['Emphysema'].tolist()\n",
    "\n",
    "labelB = (df[diseases].sum(axis=1)>0).tolist()\n",
    "labelB = np.array(labelB, dtype=int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the call bellow, I load, convert images to numpy array and split dataset to training, validation and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/aind2/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n",
      "100%|██████████| 89600/89600 [19:36<00:00, 76.18it/s]\n",
      "100%|██████████| 11200/11200 [02:26<00:00, 76.22it/s]\n",
      "100%|██████████| 11304/11304 [02:27<00:00, 76.46it/s]\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing import image                  \n",
    "from tqdm import tqdm\n",
    "\n",
    "def path_to_tensor(img_path, shape):\n",
    "    # loads RGB image as PIL.Image.Image type\n",
    "    img = image.load_img(img_path, grayscale=True, target_size=shape)\n",
    "    # convert PIL.Image.Image type to 3D tensor with shape (224, 224, 1)\n",
    "    x = image.img_to_array(img)/255\n",
    "    # convert 3D tensor to 4D tensor with shape (1, 224, 224, 1) and return 4D tensor\n",
    "    return np.expand_dims(x, axis=0)\n",
    "\n",
    "def paths_to_tensor(img_paths, shape):\n",
    "    list_of_tensors = [path_to_tensor(img_path, shape) for img_path in tqdm(img_paths)]\n",
    "    return np.vstack(list_of_tensors)\n",
    "\n",
    "train_labels = labelB[:89600][:, np.newaxis]\n",
    "valid_labels = labelB[89600:100800][:, np.newaxis]\n",
    "test_labels = labelB[100800:][:, np.newaxis]\n",
    "\n",
    "train_data = data[:89600]\n",
    "valid_data = data[89600:100800]\n",
    "test_data = data[100800:]\n",
    "\n",
    "img_shape = (64, 64)\n",
    "train_tensors = paths_to_tensor(files_list[:89600], shape = img_shape)\n",
    "valid_tensors = paths_to_tensor(files_list[89600:100800], shape = img_shape)\n",
    "test_tensors = paths_to_tensor(files_list[100800:], shape = img_shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert label from 1D to 2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def onhotLabels(label):\n",
    "    from sklearn.preprocessing import OneHotEncoder\n",
    "    enc = OneHotEncoder()\n",
    "    enc.fit(label)\n",
    "    return enc.transform(label).toarray()\n",
    "\n",
    "train_labels = onhotLabels(train_labels)\n",
    "valid_labels = onhotLabels(valid_labels)\n",
    "test_labels = onhotLabels(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CapsNet model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 64, 64, 1)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                   (None, 32, 32, 256)   20992       input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "primarycap_conv2d (Conv2D)       (None, 16, 16, 256)   5308672     conv1[0][0]                      \n",
      "____________________________________________________________________________________________________\n",
      "primarycap_reshape (Reshape)     (None, 8192, 8)       0           primarycap_conv2d[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "primarycap_squash (Lambda)       (None, 8192, 8)       0           primarycap_reshape[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "digitcaps (CapsuleLayer)         (None, 2, 16)         2097152     primarycap_squash[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "input_2 (InputLayer)             (None, 2)             0                                            \n",
      "____________________________________________________________________________________________________\n",
      "mask_1 (Mask)                    (None, 32)            0           digitcaps[0][0]                  \n",
      "                                                                   input_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "capsnet (Length)                 (None, 2)             0           digitcaps[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "decoder (Sequential)             (None, 64, 64, 1)     4740608     mask_1[0][0]                     \n",
      "====================================================================================================\n",
      "Total params: 12,167,424\n",
      "Trainable params: 12,167,424\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import argparse\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras import callbacks\n",
    "\n",
    "import numpy as np\n",
    "from keras import layers, models, optimizers\n",
    "from keras import backend as K\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask\n",
    "\n",
    "def CapsNet(input_shape, n_class, routings):\n",
    "    \"\"\"\n",
    "    A Capsule Network on MNIST.\n",
    "    :param input_shape: data shape, 3d, [width, height, channels]\n",
    "    :param n_class: number of classes\n",
    "    :param routings: number of routing iterations\n",
    "    :return: Two Keras Models, the first one used for training, and the second one for evaluation.\n",
    "            `eval_model` can also be used for training.\n",
    "    \"\"\"\n",
    "    x = layers.Input(shape=input_shape)\n",
    "\n",
    "    # Layer 1: Just a conventional Conv2D layer\n",
    "    conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=2, padding='same', activation='relu', name='conv1')(x)\n",
    "\n",
    "    # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]\n",
    "    primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='same')\n",
    "\n",
    "    # Layer 3: Capsule layer. Routing algorithm works here.\n",
    "    digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,\n",
    "                             name='digitcaps')(primarycaps)\n",
    "\n",
    "    # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.\n",
    "    # If using tensorflow, this will not be necessary. :)\n",
    "    out_caps = Length(name='capsnet')(digitcaps)\n",
    "\n",
    "    # Decoder network.\n",
    "    y = layers.Input(shape=(n_class,))\n",
    "    masked_by_y = Mask()([digitcaps, y])  # The true label is used to mask the output of capsule layer. For training\n",
    "    masked = Mask()(digitcaps)  # Mask using the capsule with maximal length. For prediction\n",
    "\n",
    "    # Shared Decoder model in training and prediction\n",
    "    decoder = models.Sequential(name='decoder')\n",
    "    decoder.add(layers.Dense(512, activation='relu', input_dim=16*n_class))\n",
    "    decoder.add(layers.Dense(1024, activation='relu'))\n",
    "    decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))\n",
    "    decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))\n",
    "\n",
    "    # Models for training and evaluation (prediction)\n",
    "    train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])\n",
    "    eval_model = models.Model(x, [out_caps, decoder(masked)])\n",
    "\n",
    "    # manipulate model\n",
    "    noise = layers.Input(shape=(n_class, 16))\n",
    "    noised_digitcaps = layers.Add()([digitcaps, noise])\n",
    "    masked_noised_y = Mask()([noised_digitcaps, y])\n",
    "    manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))\n",
    "    return train_model, eval_model, manipulate_model\n",
    "\n",
    "model, eval_model, manipulate_model = CapsNet(input_shape=train_tensors.shape[1:],\n",
    "                                                  n_class=len(np.unique(train_labels)),\n",
    "                                                  routings=4)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I implement custom matrics for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "def binary_accuracy(y_true, y_pred):\n",
    "    y_true, y_pred = K.cast(K.argmax(y_true, axis=1), K.floatx()), K.cast(K.argmax(y_pred, axis=1), K.floatx())\n",
    "    return K.mean(K.equal(y_true, K.round(y_pred)))\n",
    "\n",
    "def precision_threshold(threshold = 0.5):\n",
    "    def precision(y_true, y_pred):\n",
    "        y_true, y_pred = K.cast(K.argmax(y_true, axis=1), K.floatx()), K.cast(K.argmax(y_pred, axis=1), K.floatx())\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(y_pred)\n",
    "        precision_ratio = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision_ratio\n",
    "    return precision\n",
    "\n",
    "def recall_threshold(threshold = 0.5):\n",
    "    def recall(y_true, y_pred):\n",
    "        y_true, y_pred = K.cast(K.argmax(y_true, axis=1), K.floatx()), K.cast(K.argmax(y_pred, axis=1), K.floatx())\n",
    "        threshold_value = threshold\n",
    "        y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), threshold_value), K.floatx())\n",
    "        true_positives = K.round(K.sum(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.clip(y_true, 0, 1))\n",
    "        recall_ratio = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall_ratio\n",
    "    return recall\n",
    "\n",
    "def fbeta_score_threshold(beta = 1, threshold = 0.5):\n",
    "    def fbeta_score(y_true, y_pred):\n",
    "        threshold_value = threshold\n",
    "        beta_value = beta\n",
    "        p = precision_threshold(threshold_value)(y_true, y_pred)\n",
    "        r = recall_threshold(threshold_value)(y_true, y_pred)\n",
    "        bb = beta_value ** 2\n",
    "        fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
    "        return fbeta_score\n",
    "    return fbeta_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def margin_loss(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Margin loss for Eq.(4). When y_true[i, :] contains not just one `1`, this loss should work too. Not test it.\n",
    "    :param y_true: [None, n_classes]\n",
    "    :param y_pred: [None, num_capsule]\n",
    "    :return: a scalar loss value.\n",
    "    \"\"\"\n",
    "    L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \\\n",
    "        0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1))\n",
    "\n",
    "    return K.mean(K.sum(L, 1))\n",
    "\n",
    "def train(model, data, lr, lr_decay, lam_recon, batch_size, shift_fraction, epochs):\n",
    "    \"\"\"\n",
    "    Training a CapsuleNet\n",
    "    :param model: the CapsuleNet model\n",
    "    :param data: a tuple containing training and testing data, like `((x_train, y_train), (x_test, y_test))`\n",
    "    :param args: arguments\n",
    "    :return: The trained model\n",
    "    \"\"\"\n",
    "    # unpacking the data\n",
    "    (x_train, y_train), (x_test, y_test) = data\n",
    "\n",
    "    # callbacks\n",
    "    log = callbacks.CSVLogger('saved_models/CapsNet_log.csv')\n",
    "    tb = callbacks.TensorBoard(log_dir='saved_models/tensorboard-logs',\n",
    "                               batch_size=batch_size, histogram_freq=0)\n",
    "    checkpoint = callbacks.ModelCheckpoint(filepath='saved_models/CapsNet.best.from_scratch.hdf5', \n",
    "                               verbose=1, save_best_only=True)\n",
    "    cb_lr_decay = callbacks.LearningRateScheduler(schedule=lambda epoch: lr * (lr_decay ** epoch))\n",
    "\n",
    "    # compile the model\n",
    "    model.compile(optimizer='sgd', loss='binary_crossentropy', \n",
    "              metrics=[precision_threshold(threshold = 0.5), \n",
    "                      recall_threshold(threshold = 0.5), \n",
    "                      fbeta_score_threshold(beta=0.5, threshold = 0.5),\n",
    "                      binary_accuracy])\n",
    "\n",
    "    # Training without data augmentation:\n",
    "#     model.fit([x_train, y_train], [y_train, x_train], batch_size=batch_size, epochs=epochs,\n",
    "#               validation_data=[[x_test, y_test], [y_test, x_test]], callbacks=[log, tb, checkpoint, cb_lr_decay])\n",
    "\n",
    "    # Begin: Training with data augmentation ---------------------------------------------------------------------#\n",
    "    def train_generator(x, y, batch_size, shift_fraction=0.):\n",
    "        train_datagen = ImageDataGenerator(width_shift_range=shift_fraction,\n",
    "                                           height_shift_range=shift_fraction)  # shift up to 2 pixel for MNIST\n",
    "        generator = train_datagen.flow(x, y, batch_size=batch_size)\n",
    "        while 1:\n",
    "            x_batch, y_batch = generator.next()\n",
    "            yield ([x_batch, y_batch], [y_batch, x_batch])\n",
    "\n",
    "    # Training with data augmentation. If shift_fraction=0., also no augmentation.\n",
    "    model.fit_generator(generator=train_generator(x_train, y_train, batch_size, shift_fraction),\n",
    "                        steps_per_epoch=int(y_train.shape[0] / batch_size),\n",
    "                        epochs=epochs,\n",
    "                        validation_data=[[x_test, y_test], [y_test, x_test]],\n",
    "                        callbacks=[log, tb, checkpoint, cb_lr_decay])\n",
    "    # End: Training with data augmentation -----------------------------------------------------------------------#\n",
    "\n",
    "    from utils import plot_log\n",
    "    plot_log('saved_models/CapsNet_log.csv', show=True)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3833 - capsnet_loss: 0.6901 - decoder_loss: 0.6932 - capsnet_precision: 0.4934 - capsnet_recall: 0.2591 - capsnet_fbeta_score: 0.3791 - capsnet_binary_accuracy: 0.5758 - decoder_precision: 0.9667 - decoder_recall: 0.9915 - decoder_fbeta_score: 0.9715 - decoder_binary_accuracy: 0.0174Epoch 00000: val_loss improved from inf to 1.36107, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 856s - loss: 1.3833 - capsnet_loss: 0.6901 - decoder_loss: 0.6932 - capsnet_precision: 0.4935 - capsnet_recall: 0.2591 - capsnet_fbeta_score: 0.3792 - capsnet_binary_accuracy: 0.5758 - decoder_precision: 0.9667 - decoder_recall: 0.9915 - decoder_fbeta_score: 0.9715 - decoder_binary_accuracy: 0.0174 - val_loss: 1.3611 - val_capsnet_loss: 0.6679 - val_decoder_loss: 0.6931 - val_capsnet_precision: 0.6246 - val_capsnet_recall: 0.3468 - val_capsnet_fbeta_score: 0.5231 - val_capsnet_binary_accuracy: 0.6002 - val_decoder_precision: 0.9591 - val_decoder_recall: 0.9914 - val_decoder_fbeta_score: 0.9654 - val_decoder_binary_accuracy: 0.0164\n",
      "Epoch 2/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3616 - capsnet_loss: 0.6684 - decoder_loss: 0.6931 - capsnet_precision: 0.6013 - capsnet_recall: 0.4278 - capsnet_fbeta_score: 0.5392 - capsnet_binary_accuracy: 0.6023 - decoder_precision: 0.9665 - decoder_recall: 0.9905 - decoder_fbeta_score: 0.9712 - decoder_binary_accuracy: 0.0172Epoch 00001: val_loss improved from 1.36107 to 1.35622, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 859s - loss: 1.3615 - capsnet_loss: 0.6684 - decoder_loss: 0.6931 - capsnet_precision: 0.6013 - capsnet_recall: 0.4277 - capsnet_fbeta_score: 0.5392 - capsnet_binary_accuracy: 0.6023 - decoder_precision: 0.9665 - decoder_recall: 0.9905 - decoder_fbeta_score: 0.9712 - decoder_binary_accuracy: 0.0172 - val_loss: 1.3562 - val_capsnet_loss: 0.6631 - val_decoder_loss: 0.6931 - val_capsnet_precision: 0.6344 - val_capsnet_recall: 0.3322 - val_capsnet_fbeta_score: 0.5212 - val_capsnet_binary_accuracy: 0.6023 - val_decoder_precision: 0.9591 - val_decoder_recall: 0.9908 - val_decoder_fbeta_score: 0.9652 - val_decoder_binary_accuracy: 0.0178\n",
      "Epoch 3/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3574 - capsnet_loss: 0.6643 - decoder_loss: 0.6931 - capsnet_precision: 0.6022 - capsnet_recall: 0.4494 - capsnet_fbeta_score: 0.5506 - capsnet_binary_accuracy: 0.6077 - decoder_precision: 0.9666 - decoder_recall: 0.9905 - decoder_fbeta_score: 0.9712 - decoder_binary_accuracy: 0.0183Epoch 00002: val_loss improved from 1.35622 to 1.35307, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 859s - loss: 1.3574 - capsnet_loss: 0.6643 - decoder_loss: 0.6931 - capsnet_precision: 0.6022 - capsnet_recall: 0.4493 - capsnet_fbeta_score: 0.5506 - capsnet_binary_accuracy: 0.6077 - decoder_precision: 0.9666 - decoder_recall: 0.9905 - decoder_fbeta_score: 0.9712 - decoder_binary_accuracy: 0.0183 - val_loss: 1.3531 - val_capsnet_loss: 0.6600 - val_decoder_loss: 0.6931 - val_capsnet_precision: 0.6306 - val_capsnet_recall: 0.3651 - val_capsnet_fbeta_score: 0.5373 - val_capsnet_binary_accuracy: 0.6079 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9892 - val_decoder_fbeta_score: 0.9651 - val_decoder_binary_accuracy: 0.0186\n",
      "Epoch 4/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3558 - capsnet_loss: 0.6627 - decoder_loss: 0.6931 - capsnet_precision: 0.6014 - capsnet_recall: 0.4574 - capsnet_fbeta_score: 0.5533 - capsnet_binary_accuracy: 0.6085 - decoder_precision: 0.9669 - decoder_recall: 0.9889 - decoder_fbeta_score: 0.9712 - decoder_binary_accuracy: 0.0189Epoch 00003: val_loss improved from 1.35307 to 1.35017, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 859s - loss: 1.3558 - capsnet_loss: 0.6627 - decoder_loss: 0.6931 - capsnet_precision: 0.6014 - capsnet_recall: 0.4573 - capsnet_fbeta_score: 0.5533 - capsnet_binary_accuracy: 0.6085 - decoder_precision: 0.9669 - decoder_recall: 0.9889 - decoder_fbeta_score: 0.9712 - decoder_binary_accuracy: 0.0189 - val_loss: 1.3502 - val_capsnet_loss: 0.6571 - val_decoder_loss: 0.6931 - val_capsnet_precision: 0.6174 - val_capsnet_recall: 0.4616 - val_capsnet_fbeta_score: 0.5684 - val_capsnet_binary_accuracy: 0.6185 - val_decoder_precision: 0.9593 - val_decoder_recall: 0.9899 - val_decoder_fbeta_score: 0.9652 - val_decoder_binary_accuracy: 0.0189\n",
      "Epoch 5/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3551 - capsnet_loss: 0.6620 - decoder_loss: 0.6931 - capsnet_precision: 0.6031 - capsnet_recall: 0.4667 - capsnet_fbeta_score: 0.5585 - capsnet_binary_accuracy: 0.6105 - decoder_precision: 0.9672 - decoder_recall: 0.9888 - decoder_fbeta_score: 0.9714 - decoder_binary_accuracy: 0.0191Epoch 00004: val_loss did not improve\n",
      "2800/2800 [==============================] - 859s - loss: 1.3550 - capsnet_loss: 0.6620 - decoder_loss: 0.6931 - capsnet_precision: 0.6031 - capsnet_recall: 0.4667 - capsnet_fbeta_score: 0.5585 - capsnet_binary_accuracy: 0.6105 - decoder_precision: 0.9672 - decoder_recall: 0.9888 - decoder_fbeta_score: 0.9714 - decoder_binary_accuracy: 0.0191 - val_loss: 1.3520 - val_capsnet_loss: 0.6589 - val_decoder_loss: 0.6931 - val_capsnet_precision: 0.6369 - val_capsnet_recall: 0.3564 - val_capsnet_fbeta_score: 0.5360 - val_capsnet_binary_accuracy: 0.6082 - val_decoder_precision: 0.9595 - val_decoder_recall: 0.9882 - val_decoder_fbeta_score: 0.9650 - val_decoder_binary_accuracy: 0.0193\n",
      "Epoch 6/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3540 - capsnet_loss: 0.6609 - decoder_loss: 0.6931 - capsnet_precision: 0.6024 - capsnet_recall: 0.4714 - capsnet_fbeta_score: 0.5593 - capsnet_binary_accuracy: 0.6112 - decoder_precision: 0.9670 - decoder_recall: 0.9879 - decoder_fbeta_score: 0.9711 - decoder_binary_accuracy: 0.0196Epoch 00005: val_loss improved from 1.35017 to 1.34902, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 860s - loss: 1.3540 - capsnet_loss: 0.6609 - decoder_loss: 0.6931 - capsnet_precision: 0.6023 - capsnet_recall: 0.4713 - capsnet_fbeta_score: 0.5592 - capsnet_binary_accuracy: 0.6111 - decoder_precision: 0.9670 - decoder_recall: 0.9879 - decoder_fbeta_score: 0.9711 - decoder_binary_accuracy: 0.0196 - val_loss: 1.3490 - val_capsnet_loss: 0.6559 - val_decoder_loss: 0.6931 - val_capsnet_precision: 0.6237 - val_capsnet_recall: 0.4471 - val_capsnet_fbeta_score: 0.5677 - val_capsnet_binary_accuracy: 0.6201 - val_decoder_precision: 0.9595 - val_decoder_recall: 0.9882 - val_decoder_fbeta_score: 0.9651 - val_decoder_binary_accuracy: 0.0189\n",
      "Epoch 7/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3530 - capsnet_loss: 0.6600 - decoder_loss: 0.6931 - capsnet_precision: 0.6020 - capsnet_recall: 0.4763 - capsnet_fbeta_score: 0.5618 - capsnet_binary_accuracy: 0.6119 - decoder_precision: 0.9668 - decoder_recall: 0.9890 - decoder_fbeta_score: 0.9711 - decoder_binary_accuracy: 0.0198Epoch 00006: val_loss improved from 1.34902 to 1.34873, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 857s - loss: 1.3530 - capsnet_loss: 0.6600 - decoder_loss: 0.6931 - capsnet_precision: 0.6020 - capsnet_recall: 0.4763 - capsnet_fbeta_score: 0.5617 - capsnet_binary_accuracy: 0.6119 - decoder_precision: 0.9669 - decoder_recall: 0.9890 - decoder_fbeta_score: 0.9711 - decoder_binary_accuracy: 0.0198 - val_loss: 1.3487 - val_capsnet_loss: 0.6557 - val_decoder_loss: 0.6931 - val_capsnet_precision: 0.6138 - val_capsnet_recall: 0.4916 - val_capsnet_fbeta_score: 0.5761 - val_capsnet_binary_accuracy: 0.6221 - val_decoder_precision: 0.9595 - val_decoder_recall: 0.9906 - val_decoder_fbeta_score: 0.9655 - val_decoder_binary_accuracy: 0.0196\n",
      "Epoch 8/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3518 - capsnet_loss: 0.6588 - decoder_loss: 0.6931 - capsnet_precision: 0.6040 - capsnet_recall: 0.4810 - capsnet_fbeta_score: 0.5648 - capsnet_binary_accuracy: 0.6149 - decoder_precision: 0.9671 - decoder_recall: 0.9908 - decoder_fbeta_score: 0.9717 - decoder_binary_accuracy: 0.0202Epoch 00007: val_loss improved from 1.34873 to 1.34814, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 855s - loss: 1.3518 - capsnet_loss: 0.6588 - decoder_loss: 0.6931 - capsnet_precision: 0.6040 - capsnet_recall: 0.4811 - capsnet_fbeta_score: 0.5648 - capsnet_binary_accuracy: 0.6149 - decoder_precision: 0.9671 - decoder_recall: 0.9908 - decoder_fbeta_score: 0.9717 - decoder_binary_accuracy: 0.0202 - val_loss: 1.3481 - val_capsnet_loss: 0.6551 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6265 - val_capsnet_recall: 0.4312 - val_capsnet_fbeta_score: 0.5640 - val_capsnet_binary_accuracy: 0.6190 - val_decoder_precision: 0.9595 - val_decoder_recall: 0.9918 - val_decoder_fbeta_score: 0.9657 - val_decoder_binary_accuracy: 0.0203\n",
      "Epoch 9/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3524 - capsnet_loss: 0.6593 - decoder_loss: 0.6931 - capsnet_precision: 0.6034 - capsnet_recall: 0.4811 - capsnet_fbeta_score: 0.5647 - capsnet_binary_accuracy: 0.6145 - decoder_precision: 0.9670 - decoder_recall: 0.9917 - decoder_fbeta_score: 0.9718 - decoder_binary_accuracy: 0.0204Epoch 00008: val_loss improved from 1.34814 to 1.34727, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 857s - loss: 1.3523 - capsnet_loss: 0.6593 - decoder_loss: 0.6931 - capsnet_precision: 0.6035 - capsnet_recall: 0.4811 - capsnet_fbeta_score: 0.5647 - capsnet_binary_accuracy: 0.6145 - decoder_precision: 0.9670 - decoder_recall: 0.9917 - decoder_fbeta_score: 0.9718 - decoder_binary_accuracy: 0.0204 - val_loss: 1.3473 - val_capsnet_loss: 0.6542 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6210 - val_capsnet_recall: 0.4629 - val_capsnet_fbeta_score: 0.5720 - val_capsnet_binary_accuracy: 0.6208 - val_decoder_precision: 0.9595 - val_decoder_recall: 0.9917 - val_decoder_fbeta_score: 0.9657 - val_decoder_binary_accuracy: 0.0210\n",
      "Epoch 10/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3516 - capsnet_loss: 0.6586 - decoder_loss: 0.6930 - capsnet_precision: 0.6044 - capsnet_recall: 0.4831 - capsnet_fbeta_score: 0.5662 - capsnet_binary_accuracy: 0.6150 - decoder_precision: 0.9672 - decoder_recall: 0.9924 - decoder_fbeta_score: 0.9721 - decoder_binary_accuracy: 0.0208Epoch 00009: val_loss improved from 1.34727 to 1.34695, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 857s - loss: 1.3516 - capsnet_loss: 0.6586 - decoder_loss: 0.6930 - capsnet_precision: 0.6044 - capsnet_recall: 0.4831 - capsnet_fbeta_score: 0.5662 - capsnet_binary_accuracy: 0.6150 - decoder_precision: 0.9672 - decoder_recall: 0.9924 - decoder_fbeta_score: 0.9721 - decoder_binary_accuracy: 0.0208 - val_loss: 1.3469 - val_capsnet_loss: 0.6539 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6166 - val_capsnet_recall: 0.4883 - val_capsnet_fbeta_score: 0.5767 - val_capsnet_binary_accuracy: 0.6228 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9930 - val_decoder_fbeta_score: 0.9659 - val_decoder_binary_accuracy: 0.0215\n",
      "Epoch 11/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3511 - capsnet_loss: 0.6580 - decoder_loss: 0.6930 - capsnet_precision: 0.6026 - capsnet_recall: 0.4840 - capsnet_fbeta_score: 0.5657 - capsnet_binary_accuracy: 0.6153 - decoder_precision: 0.9666 - decoder_recall: 0.9933 - decoder_fbeta_score: 0.9718 - decoder_binary_accuracy: 0.0210Epoch 00010: val_loss did not improve\n",
      "2800/2800 [==============================] - 856s - loss: 1.3510 - capsnet_loss: 0.6580 - decoder_loss: 0.6930 - capsnet_precision: 0.6027 - capsnet_recall: 0.4840 - capsnet_fbeta_score: 0.5657 - capsnet_binary_accuracy: 0.6153 - decoder_precision: 0.9666 - decoder_recall: 0.9933 - decoder_fbeta_score: 0.9718 - decoder_binary_accuracy: 0.0210 - val_loss: 1.3480 - val_capsnet_loss: 0.6550 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6363 - val_capsnet_recall: 0.4056 - val_capsnet_fbeta_score: 0.5596 - val_capsnet_binary_accuracy: 0.6187 - val_decoder_precision: 0.9595 - val_decoder_recall: 0.9931 - val_decoder_fbeta_score: 0.9659 - val_decoder_binary_accuracy: 0.0223\n",
      "Epoch 12/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3512 - capsnet_loss: 0.6582 - decoder_loss: 0.6930 - capsnet_precision: 0.6049 - capsnet_recall: 0.4851 - capsnet_fbeta_score: 0.5674 - capsnet_binary_accuracy: 0.6156 - decoder_precision: 0.9668 - decoder_recall: 0.9941 - decoder_fbeta_score: 0.9721 - decoder_binary_accuracy: 0.0211Epoch 00011: val_loss did not improve\n",
      "2800/2800 [==============================] - 857s - loss: 1.3512 - capsnet_loss: 0.6582 - decoder_loss: 0.6930 - capsnet_precision: 0.6048 - capsnet_recall: 0.4851 - capsnet_fbeta_score: 0.5674 - capsnet_binary_accuracy: 0.6156 - decoder_precision: 0.9668 - decoder_recall: 0.9941 - decoder_fbeta_score: 0.9721 - decoder_binary_accuracy: 0.0211 - val_loss: 1.3470 - val_capsnet_loss: 0.6540 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6281 - val_capsnet_recall: 0.4259 - val_capsnet_fbeta_score: 0.5625 - val_capsnet_binary_accuracy: 0.6184 - val_decoder_precision: 0.9595 - val_decoder_recall: 0.9936 - val_decoder_fbeta_score: 0.9660 - val_decoder_binary_accuracy: 0.0222\n",
      "Epoch 13/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3508 - capsnet_loss: 0.6577 - decoder_loss: 0.6930 - capsnet_precision: 0.6064 - capsnet_recall: 0.4866 - capsnet_fbeta_score: 0.5688 - capsnet_binary_accuracy: 0.6168 - decoder_precision: 0.9669 - decoder_recall: 0.9941 - decoder_fbeta_score: 0.9722 - decoder_binary_accuracy: 0.0215Epoch 00012: val_loss improved from 1.34695 to 1.34649, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 857s - loss: 1.3508 - capsnet_loss: 0.6577 - decoder_loss: 0.6930 - capsnet_precision: 0.6063 - capsnet_recall: 0.4867 - capsnet_fbeta_score: 0.5687 - capsnet_binary_accuracy: 0.6168 - decoder_precision: 0.9669 - decoder_recall: 0.9941 - decoder_fbeta_score: 0.9722 - decoder_binary_accuracy: 0.0215 - val_loss: 1.3465 - val_capsnet_loss: 0.6535 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6277 - val_capsnet_recall: 0.4492 - val_capsnet_fbeta_score: 0.5710 - val_capsnet_binary_accuracy: 0.6217 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9937 - val_decoder_fbeta_score: 0.9660 - val_decoder_binary_accuracy: 0.0224\n",
      "Epoch 14/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3509 - capsnet_loss: 0.6579 - decoder_loss: 0.6930 - capsnet_precision: 0.6036 - capsnet_recall: 0.4852 - capsnet_fbeta_score: 0.5669 - capsnet_binary_accuracy: 0.6155 - decoder_precision: 0.9668 - decoder_recall: 0.9945 - decoder_fbeta_score: 0.9721 - decoder_binary_accuracy: 0.0216Epoch 00013: val_loss did not improve\n",
      "2800/2800 [==============================] - 855s - loss: 1.3509 - capsnet_loss: 0.6579 - decoder_loss: 0.6930 - capsnet_precision: 0.6036 - capsnet_recall: 0.4853 - capsnet_fbeta_score: 0.5670 - capsnet_binary_accuracy: 0.6156 - decoder_precision: 0.9668 - decoder_recall: 0.9945 - decoder_fbeta_score: 0.9721 - decoder_binary_accuracy: 0.0216 - val_loss: 1.3488 - val_capsnet_loss: 0.6558 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6471 - val_capsnet_recall: 0.3819 - val_capsnet_fbeta_score: 0.5552 - val_capsnet_binary_accuracy: 0.6176 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9947 - val_decoder_fbeta_score: 0.9662 - val_decoder_binary_accuracy: 0.0230\n",
      "Epoch 15/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3497 - capsnet_loss: 0.6567 - decoder_loss: 0.6930 - capsnet_precision: 0.6057 - capsnet_recall: 0.4853 - capsnet_fbeta_score: 0.5678 - capsnet_binary_accuracy: 0.6160 - decoder_precision: 0.9666 - decoder_recall: 0.9956 - decoder_fbeta_score: 0.9722 - decoder_binary_accuracy: 0.0219Epoch 00014: val_loss improved from 1.34649 to 1.34594, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 854s - loss: 1.3497 - capsnet_loss: 0.6566 - decoder_loss: 0.6930 - capsnet_precision: 0.6056 - capsnet_recall: 0.4854 - capsnet_fbeta_score: 0.5678 - capsnet_binary_accuracy: 0.6161 - decoder_precision: 0.9666 - decoder_recall: 0.9956 - decoder_fbeta_score: 0.9722 - decoder_binary_accuracy: 0.0219 - val_loss: 1.3459 - val_capsnet_loss: 0.6529 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6253 - val_capsnet_recall: 0.4580 - val_capsnet_fbeta_score: 0.5727 - val_capsnet_binary_accuracy: 0.6220 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9962 - val_decoder_fbeta_score: 0.9665 - val_decoder_binary_accuracy: 0.0223\n",
      "Epoch 16/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3495 - capsnet_loss: 0.6565 - decoder_loss: 0.6930 - capsnet_precision: 0.6059 - capsnet_recall: 0.4849 - capsnet_fbeta_score: 0.5693 - capsnet_binary_accuracy: 0.6167 - decoder_precision: 0.9669 - decoder_recall: 0.9962 - decoder_fbeta_score: 0.9725 - decoder_binary_accuracy: 0.0219Epoch 00015: val_loss improved from 1.34594 to 1.34578, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 854s - loss: 1.3495 - capsnet_loss: 0.6565 - decoder_loss: 0.6930 - capsnet_precision: 0.6059 - capsnet_recall: 0.4849 - capsnet_fbeta_score: 0.5692 - capsnet_binary_accuracy: 0.6167 - decoder_precision: 0.9669 - decoder_recall: 0.9962 - decoder_fbeta_score: 0.9725 - decoder_binary_accuracy: 0.0219 - val_loss: 1.3458 - val_capsnet_loss: 0.6528 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6254 - val_capsnet_recall: 0.4622 - val_capsnet_fbeta_score: 0.5743 - val_capsnet_binary_accuracy: 0.6229 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9966 - val_decoder_fbeta_score: 0.9665 - val_decoder_binary_accuracy: 0.0229\n",
      "Epoch 17/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3496 - capsnet_loss: 0.6566 - decoder_loss: 0.6930 - capsnet_precision: 0.6075 - capsnet_recall: 0.4893 - capsnet_fbeta_score: 0.5710 - capsnet_binary_accuracy: 0.6179 - decoder_precision: 0.9669 - decoder_recall: 0.9964 - decoder_fbeta_score: 0.9726 - decoder_binary_accuracy: 0.0220Epoch 00016: val_loss did not improve\n",
      "2800/2800 [==============================] - 854s - loss: 1.3497 - capsnet_loss: 0.6566 - decoder_loss: 0.6930 - capsnet_precision: 0.6075 - capsnet_recall: 0.4893 - capsnet_fbeta_score: 0.5710 - capsnet_binary_accuracy: 0.6179 - decoder_precision: 0.9669 - decoder_recall: 0.9964 - decoder_fbeta_score: 0.9726 - decoder_binary_accuracy: 0.0220 - val_loss: 1.3471 - val_capsnet_loss: 0.6541 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6391 - val_capsnet_recall: 0.4042 - val_capsnet_fbeta_score: 0.5605 - val_capsnet_binary_accuracy: 0.6188 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9971 - val_decoder_fbeta_score: 0.9666 - val_decoder_binary_accuracy: 0.0234\n",
      "Epoch 18/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3492 - capsnet_loss: 0.6562 - decoder_loss: 0.6930 - capsnet_precision: 0.6089 - capsnet_recall: 0.4913 - capsnet_fbeta_score: 0.5725 - capsnet_binary_accuracy: 0.6188 - decoder_precision: 0.9671 - decoder_recall: 0.9966 - decoder_fbeta_score: 0.9728 - decoder_binary_accuracy: 0.0222Epoch 00017: val_loss improved from 1.34578 to 1.34566, saving model to saved_models/CapsNet.best.from_scratch.hdf5\n",
      "2800/2800 [==============================] - 856s - loss: 1.3492 - capsnet_loss: 0.6562 - decoder_loss: 0.6930 - capsnet_precision: 0.6088 - capsnet_recall: 0.4913 - capsnet_fbeta_score: 0.5725 - capsnet_binary_accuracy: 0.6187 - decoder_precision: 0.9671 - decoder_recall: 0.9966 - decoder_fbeta_score: 0.9728 - decoder_binary_accuracy: 0.0222 - val_loss: 1.3457 - val_capsnet_loss: 0.6527 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6247 - val_capsnet_recall: 0.4630 - val_capsnet_fbeta_score: 0.5741 - val_capsnet_binary_accuracy: 0.6229 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9969 - val_decoder_fbeta_score: 0.9666 - val_decoder_binary_accuracy: 0.0225\n",
      "Epoch 19/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3495 - capsnet_loss: 0.6565 - decoder_loss: 0.6930 - capsnet_precision: 0.6074 - capsnet_recall: 0.4896 - capsnet_fbeta_score: 0.5709 - capsnet_binary_accuracy: 0.6186 - decoder_precision: 0.9669 - decoder_recall: 0.9964 - decoder_fbeta_score: 0.9726 - decoder_binary_accuracy: 0.0223Epoch 00018: val_loss did not improve\n",
      "2800/2800 [==============================] - 857s - loss: 1.3495 - capsnet_loss: 0.6565 - decoder_loss: 0.6930 - capsnet_precision: 0.6074 - capsnet_recall: 0.4896 - capsnet_fbeta_score: 0.5709 - capsnet_binary_accuracy: 0.6186 - decoder_precision: 0.9669 - decoder_recall: 0.9964 - decoder_fbeta_score: 0.9726 - decoder_binary_accuracy: 0.0223 - val_loss: 1.3457 - val_capsnet_loss: 0.6527 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6315 - val_capsnet_recall: 0.4409 - val_capsnet_fbeta_score: 0.5705 - val_capsnet_binary_accuracy: 0.6225 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9968 - val_decoder_fbeta_score: 0.9666 - val_decoder_binary_accuracy: 0.0238\n",
      "Epoch 20/20\n",
      "2799/2800 [============================>.] - ETA: 0s - loss: 1.3491 - capsnet_loss: 0.6561 - decoder_loss: 0.6930 - capsnet_precision: 0.6060 - capsnet_recall: 0.4901 - capsnet_fbeta_score: 0.5706 - capsnet_binary_accuracy: 0.6184 - decoder_precision: 0.9669 - decoder_recall: 0.9969 - decoder_fbeta_score: 0.9727 - decoder_binary_accuracy: 0.0225Epoch 00019: val_loss did not improve\n",
      "2800/2800 [==============================] - 858s - loss: 1.3491 - capsnet_loss: 0.6561 - decoder_loss: 0.6930 - capsnet_precision: 0.6061 - capsnet_recall: 0.4901 - capsnet_fbeta_score: 0.5707 - capsnet_binary_accuracy: 0.6185 - decoder_precision: 0.9669 - decoder_recall: 0.9969 - decoder_fbeta_score: 0.9727 - decoder_binary_accuracy: 0.0225 - val_loss: 1.3460 - val_capsnet_loss: 0.6530 - val_decoder_loss: 0.6930 - val_capsnet_precision: 0.6350 - val_capsnet_recall: 0.4251 - val_capsnet_fbeta_score: 0.5667 - val_capsnet_binary_accuracy: 0.6217 - val_decoder_precision: 0.9594 - val_decoder_recall: 0.9970 - val_decoder_fbeta_score: 0.9666 - val_decoder_binary_accuracy: 0.0236\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAG0CAYAAAD3mQiqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl8FPX9+PHXe49chDMgKogBRVBCEkICKKcn4IGAIh5VwSKlKvKtteqv1Epb+db7wLYiWlHwAK0CHrX9aoWC5ZBAA8rhgYCC3IEc5Nydz++PmV02IccCE5LA+wn72NmZz37mPbO778z5+YgxBqWUcpOnvgNQSp14NLEopVyniUUp5TpNLEop12liUUq5ThOLUsp1mlhUmIh4RaRQRDq4WfYo4nhYRF5xu151/PjqOwB19ESkMOJlAlAKBJ3XPzPGvH4k9RljgkCi22XVyUcTSyNmjAn/sEVkCzDOGPNJdeVFxGeMCRyP2NTJTXeFTmDOLsVcEXlTRAqAn4jI+SKyXEQOiMgOEZkmIn6nvE9EjIgkO69fc6Z/JCIFIrJMRDoeaVln+lAR+VpE8kTkORH5j4iMiXI5RojIOifmT0WkS8S0X4vIjyKSLyIbRWSQM76PiKx2xu8SkcddWKUqSppYTnwjgDeA5sBcIABMAloDfYEhwM9qeP+NwINAK+B74A9HWlZETgHeAn7lzHcz0Cua4EXkXGA2MBFoA3wCvCcifhHp5sSeYYxpBgx15gvwHPC4M/5s4G/RzE+5QxPLie8zY8z7xhjLGFNsjFlpjFlhjAkYY74DZgADa3j/34wx2caYcuB1IP0oyl4J5BhjFjjTngb2Rhn/9cB7xphPnfc+gp0ke2MnyTigm7Obt9lZJoByoLOIJBljCowxK6Kcn3KBJpYT3w+RL0Skq4h8KCI7RSQf+D32VkR1dkYMF1HzAdvqyp4eGYex73zdFkXsofdujXiv5by3nTHmK+CX2Muw29nlO9UpOhY4D/hKRD4XkcujnJ9ygSaWE1/l29dfAL4EznZ2E34LSB3HsANoH3ohIgK0i/K9PwJnRrzX49S1HcAY85oxpi/QEfACf3TGf2WMuR44BXgSeEdE4o59UVQ0NLGcfJoCecBB5/hFTcdX3PIBkCEiV4mID/sYT5so3/sWMExEBjkHmX8FFAArRORcEblQRGKBYudhAYjIzSLS2tnCycNOsJa7i6Wqo4nl5PNL4FbsH+cL2Ad065QxZhcwGngK2AecBfwX+7qb2t67Djve54E92AebhznHW2KBx7CP1+wEWgKTnbdeDmxwzoY9AYw2xpS5uFiqBqINPanjTUS82Ls41xpjltR3PMp9usWijgsRGSIiLZzdlgexz9p8Xs9hqTqiiUUdL/2A77B3ZwYDI4wxte4KqcZJd4WUUq7TLRallOs0sSilXFdvdze3bt3aJCcn19fslVJHYdWqVXuNMbVeg1RrYhGRl7Hv9dhtjEmpoVwWsAy43hhT6w1fycnJZGdn11ZMKdWAiMjW2ktFtyv0CvZFSTXNzAs8CvxfNDNVSp3Yak0sxpjFQG4txSYC7wC73QhKKdW4HfPBWxFph93mx/NRlB0vItkikr1nz55jnbVSqoFy46zQM8D9zs1eNTLGzDDGZBpjMtu0ifYeNKVUY+PGWaFMYI59JzytgctFJGCMme9C3UqpRuiYE4sxJrJd01eAD9xMKk9lP0VpsJTru15Px+Yda3+DUqreRXO6+U1gENBaRLYBDwF+AGPM9DqNDigsL2Tet/N4Y+MbnH/a+dzQ9QYGtB+A1+Ot61krpY5Svd0rlJmZaaK9jmVv8V7e/eZd5n41l91Fu2mX2I7RXUYz4uwRtIhrUceRKqVCRGSVMSaz1nKNIbGElFvlLPx+IW9ufJPsXdnEemO5vOPl3ND1Bs5NOreOIlVKhZyQiSXS1/u/Zs7GOXzw3QcUB4pJb5POjefeyCUdLsHv9bsYqVIq5IRPLCH5Zfks+HYBczbO4fuC72nqb0rbJm1pEduClnEtaRnbkhZxLWgV18oeF9vSHh/XkhaxLYj1xuKc0VJK1eKkSSwhlrFY+uNSFn6/kNySXHJLcjlQeoD9Jfs5UHoAc1hj9TaPeIj1xhLjjSHW4zx7Dz2HhuN8cfg8PnziwyMefB4fXvHi9Xjt54jhyGl+jx+fx4ff4w8Phx4VXsvh40J1VVXe7/HjEb05XR1f0SaWE6bvZo946NeuH/3a9TtsWtAKUlBWQG5pLgdK7GSzv3Q/+0v2UxwopixYRmmwlDLLeXZeh4YLygsoC5ZRFiwjaIL2wwoePhwx7njwijecsPxePz7x4ff6D0tmBkPQBDHGfraMVeEROU1EDkuUoeFQUo0c7/F47Gfx4MGe5hGPXc4ZH3oOMRiMMRWeAUJ/5EKvQ/WEH3gQkXCdIoIHT7juUH3h4Ur1VlhvznqKXH9VDfs89k9EEESE0D/7v72lGzk+NN8Ky2KoMD4Ul2AvS2ide8QTHlf5IQgWFhj7j6iFFa7LMpWGMYfqrqK+yPUmIiT4EmjftD1uOmESS028Hi8t4lrYZ5Ca1/38jDEETICgFaTcKidgBcKP0OvKzwETqFCu2nHO+HKrnPLgoTrCjyrGhb5IoR9jVc+hL50x9pczFH9o2DIWQSsYHg5YAUpMCZZlhZNVZNKqahg47IcZ+aOsvEtqjMHCwrLsH1LoBxRKhKFxlrEq/MhDIusMTQcImkOfi4LMtpnMHDLT1TpPisRyvIkIfrH/4sWhfWQ1VMYYAlaAMquM8mD5YQm6zCojaAWr3AqpPBy5qx2ZMIEqh0NbN5FbjqHX4cQZuWWJVWErI3JrJ3KrJDT/UGyV64ucZyg5t4h1/5INTSzqpCUi9m6P1+9c8qncokf/lFKu08SilHKdJhallOs0sSilXKeJRSnluloTi4i8LCK7ReTLaqZfLSJrRSTHaXby8CvUlFInFTda6f8XkGaMSQduA15yIS6lVCN2zK30G2MKzaFrpptANTflKKVOGq4cYxGRESKyEfgQe6ulunLaSr9SJwFXEosxZp4xpiswHPhDDeW0lX6lTgKunhVydps6iUhrN+tVSjUubnRYdrY4t5CKSAYQC+w71nqVUo2XG630XwPcIiLlQDEw2lRuAEMpdVKpNbEYY26oZfqj2B3CK6UUoFfeKqXqgCYWpZTrNLEopVyniUUp5TpNLEop12liUUq5ThOLUsp1mliUUq7TxKKUcp0mFqWU6zSxKKVcp4lFKeU6TSxKKde50Ur/TU4r/V+IyFIRSXM/TKVUY+JGK/2bgYHGmO7YzVLOcCEupVQjFk17LItFJLmG6UsjXi4H2h97WEqpxsztYyw/BT5yuU6lVCNT6xZLtETkQuzEUm1PiCIyHhgP0KFDB7dmrZRqYNzqVygVuwfEq40x1Takrd1/KHVycKOV/g7Au8DNxpivjz0kpVRj50Yr/b8FkoC/OL2ABIwxmXUVsFKq4XOjlf5xwDjXIlJKNXp65a1SynWaWJRSrtPEopRynSYWpZTrNLEopVyniUUp5TpNLEop12liUUq5ThOLUsp1rt3drNSxKC8vZ9u2bZSUlNR3KAqIi4ujffv2+P3+o3q/JhbVIGzbto2mTZuSnJyMc8+ZqifGGPbt28e2bdvo2LHjUdWhu0KqQSgpKSEpKUmTSgMgIiQlJR3T1qMmFtVgaFJpOI71s3Cjlf6uIrJMREpF5N5jikYpdUJwo5X+XOBu4Ak3AlJKNX61JhZjzGLs5FHd9N3GmJVAuZuBKXWyWrRoEUuXLq2xzJQpU3jiiYb7t1yPsSjVwESTWBq643q6WVvpV9H43fvrWP9jvqt1nnd6Mx66qluNZWbNmsUTTzyBiJCamsp1113Hww8/TFlZGUlJSbz++uu0bduWKVOmsGnTJr799lv27t3Lfffdx+23386OHTsYPXo0+fn5BAIBnn/+efr3709iYiKTJk3igw8+ID4+ngULFtC2bVv27NnDhAkT+P777wF45plnaNeuHdOnT8fr9fLaa6/x3HPP0b9//xrjzsnJYcKECRQVFXHWWWfx8ssv07JlS6ZNm8b06dPx+Xycd955zJkzh3//+99MmjQJsA/QLl68mKZNm7qzkiMc18RijJmB01NiZmamOZ7zVqom69at4+GHH2bp0qW0bt2a3NxcRITly5cjIrz00ks89thjPPnkkwCsXbuW5cuXc/DgQXr06MEVV1zBm2++yeDBg5k8eTLBYJCioiIADh48SJ8+fZg6dSr33XcfL774Ir/5zW+YNGkSv/jFL+jXrx/ff/89gwcPZsOGDUyYMIHExETuvTe6cyG33HILzz33HAMHDuS3v/0tv/vd73jmmWd45JFH2Lx5M7GxsRw4cACAJ554gj//+c/07duXwsJC4uLi6mR96gVyqsGpbcuiLnz66aeMGjWK1q1bA9CqVSu++OILRo8ezY4dOygrK6twsdjVV19NfHw88fHxXHjhhXz++edkZWVx2223UV5ezvDhw0lPTwcgJiaGK6+8EoCePXvy8ccfA/DJJ5+wfv36cJ35+fkUFhYeUdx5eXkcOHCAgQMHAnDrrbcyatQoAFJTU7npppsYPnw4w4cPB6Bv377cc8893HTTTYwcOZL27eum49JoTje/CSwDuojINhH5qYhMEJEJzvRTndb77wF+45RpVifRKnUcTZw4kbvuuosvvviCF154ocIFY5Wv8xARBgwYwOLFi2nXrh1jxoxh1qxZAPj9/nB5r9dLIBAAwLIsli9fTk5ODjk5OWzfvp3ExETX4v/www+58847Wb16NVlZWQQCAR544AFeeukliouL6du3Lxs3bnRtfpGiOSt0gzHmNGOM3xjT3hjzV2PMdKfrD4wxO53xzYwxLZxhd3eQlapjF110EW+//Tb79tn97eXm5pKXl0e7du0AePXVVyuUX7BgASUlJezbt49FixaRlZXF1q1badu2Lbfffjvjxo1j9erVNc7zsssu47nnngu/zsnJAaBp06YUFBREFXfz5s1p2bIlS5YsAWD27NkMHDgQy7L44YcfuPDCC3n00UfJy8ujsLCQTZs20b17d+6//36ysrLqLLHorpBSQLdu3Zg8eTIDBw7E6/XSo0cPpkyZwqhRo2jZsiUXXXQRmzdvDpdPTU3lwgsvZO/evTz44IOcfvrpvPrqqzz++OP4/X4SExPDWyzVmTZtGnfeeSepqakEAgEGDBjA9OnTueqqq7j22mtZsGBBVAdvX3311fDB206dOjFz5kyCwSA/+clPyMvLwxjD3XffTYsWLXjwwQdZuHAhHo+Hbt26MXToUFfWX2ViTP0cQ83MzDTZ2dn1Mm/V8GzYsIFzzz23vsOIypQpU47o4GpjVdVnIiKroumQUK9jUUq5TneFlDpCU6ZMOW7zmjp1Km+//XaFcaNGjWLy5MnHLYajoYlFqQZs8uTJDT6JVEV3hZRSrtPEopRynSYWpZTrNLEoVYW6bJbArbpfeeUV7rrrLhcicp8mFqUauNAtAI2JJhalHFOnTuWcc86hX79+fPXVVwBs2rSJIUOG0LNnT/r37x++BH7Xrl2MGDGCtLQ00tLSwu2nPPXUU6SkpJCSksIzzzxTY9011T9mzBgmTJhA7969ue+++2qNfcuWLVx00UWkpqZy8cUXh5tiePvtt0lJSSEtLY0BAwYA9p3cvXr1Ij09ndTUVL755hsX1l5FerpZNTwfPQA7v3C3zlO7w9BHqp28atUq5syZQ05ODoFAgIyMDHr27Mn48eOZPn06nTt3ZsWKFdxxxx18+umn3H333QwcOJB58+YRDAYpLCxk1apVzJw5kxUrVmCMoXfv3uH7dqqqG6i2frC7RFm6dCler7fWxZs4cSK33nort956Ky+//DJ333038+fP5/e//z3//Oc/adeuXbjphOnTpzNp0iRuuukmysrKCAaDLqzgijSxKAUsWbKEESNGkJCQAMCwYcMoKSlh6dKl4WYIAEpLSwG7mYXQvUBer5fmzZvz2WefMWLECJo0aQLAyJEjWbJkCZZlHVY3QGFhYbX1g30hXDRJBWDZsmW8++67ANx8883hrZy+ffsyZswYrrvuOkaOHAnA+eefz9SpU9m2bRsjR46kc+fOR7i2aldrYhGRl4Ergd3GmJQqpgvwLHA5UASMMcbUfFunUjWpYcvieLIsixYtWoTvOj7e9YcS1LGYPn06K1as4MMPP6Rnz56sWrWKG2+8kd69e/Phhx9y+eWX88ILL3DRRRcd87wiudFK/1Cgs/MYDzx/7GEpdXwNGDCA+fPnU1xcTEFBAe+//z4JCQl07NgxfEm9MYY1a9YAcPHFF/P88/ZXPRgMkpeXR//+/Zk/fz5FRUUcPHiQefPm0b9//yrrBmjWrFm19R+pCy64gDlz5gDw+uuvh++I3rRpE7179+b3v/89bdq04YcffuC7776jU6dO3H333Vx99dWsXbv26FdcNY65lX7gamCWsS0HWojIaW4FqNTxkJGRwejRo0lLS2Po0KFkZWUB9o/0r3/9K2lpaXTr1o0FCxYA8Oyzz7Jw4UK6d+9Oz549Wb9+PRkZGYwZM4ZevXrRu3dvxo0bR48ePaqtu6b6j9Rzzz3HzJkzSU1NZfbs2Tz77LMA/OpXv6J79+6kpKRwwQUXkJaWxltvvUVKSgrp6el8+eWX3HLLLce49g4XVbMJIpIMfFDNrtAHwCPGmM+c1/8C7jfG1NgmgjaboCI1pmYTThaNptkEERkvItkikr1nz57jOWul1HHkRmLZDpwR8bq9M+4wxpgZxphMY0xmmzZtXJi1Uie+mTNnkp6eXuFx55131ndYNXLjdPN7wF0iMgfoDeQZY3a4UK9SChg7dixjx46t7zCOSDSnm98EBgGtndb4HwL8AE6D2n/HPtX8Lfbp5sa1BpRSrqs1sRhjbqhlugEa9naZUuq40nuFlFKu08SilMPNzsJOdppYlFKu08SiVCXGGH71q1+RkpJC9+7dmTt3LgA7duxgwIABpKenk5KSwpIlSwgGg4wZMyZc9umnn67n6BsGvbtZqUreffddcnJyWLNmDXv37iUrK4sBAwbwxhtvMHjwYCZPnkwwGKSoqCjc5/KXX34JEG6a4GSniUU1OI9+/igbc93tU7hrq67c3+v+qMp+9tln3HDDDXi9Xtq2bcvAgQNZuXIlWVlZ3HbbbZSXlzN8+HDS09Pp1KkT3333HRMnTuSKK67gsssuczXuxkp3hZSK0oABA1i8eDHt2rVjzJgxzJo1i5YtW7JmzRoGDRrE9OnTGTduXH2H2SDoFotqcKLdsqgr/fv354UXXuDWW28lNzeXxYsX8/jjj7N161bat2/P7bffTmlpKatXr+byyy8nJiaGa665hi5duvCTn/ykXmNvKDSxKFXJiBEjWLZsGWlpaYgIjz32GKeeeiqvvvoqjz/+OH6/n8TERGbNmsX27dsZO3YslmUB8Mc//rGeo28Yomo2oS5oswkqkjab0PA0mmYTlFInB00sSinXaWJRSrkuqsQiIkNE5CsR+VZEHqhieksRmScia0XkcxE5rAlLpWpTX8f71OGO9bOoNbGIiBf4M3Zr/OcBN4jIeZWK/RrIMcakArdgdweiVNTi4uLYt2+fJpcGwBjDvn37iIuLO+o6ojnd3Av41hjzHYDTUtzVwPqIMucBjzhBbRSRZBFpa4zZddSRqZNK+/bt2bZtG9oWcsMQFxdH+/btj/r90SSWdsAPEa+3YTdBGWkNMBJYIiK9gDOx277VxKKi4vf76dixY32HoVzi1sHbR7D7E8oBJgL/BQ7rEFZb6Vfq5BDNFkutrfAbY/Jx2rp1ulzdDHxXuSJjzAxgBtgXyB1dyEqphi6aLZaVQGcR6SgiMcD12C3zh4lIC2cawDhgsZNslFInoWga0w6IyF3APwEv8LIxZp2ITHCmTwfOBV4VEQOsA35ahzErpRq4qG5CNMb8Hbubj8hx0yOGlwHnuBuaUqqx0itvlVKu08SilHKdJhallOs0sSilXKeJRSnlOk0sSinXaWJRSrlOE4tSynWaWJRSrtPEopRynSYWpZTrNLEopVyniUUp5Tq3WulvLiLvi8gaEVknImPdD1Up1Vi41Ur/ncB6Y0waMAh4MqLhJ6XUSSaaLZZwK/3GmDIg1Ep/JAM0dZqlTARygYCrkSqlGo1oEktVrfS3q1TmT9ityP0IfAFMMsZYrkSolGp03Dp4OxjIAU4H0oE/iUizyoW0lX6lTg7RJJZaW+nHbqH/XWP7FruV/q6VKzLGzDDGZBpjMtu0aXO0MSulGjhXWukHvgcuBhCRtkAXquj+Qyl1cnCrlf4/AK+IyBeAAPcbY/bWYdxKqQbMrVb6fwQuczc0pVRjpVfeKqVcp4lFKeU6TSxKKddpYlFKuU4Ti1LKdZpYlFKu08SilHKdJhallOs0sSilXKeJRSnlOk0sSinXaWJRSrlOE4tSynVutdL/KxHJcR5fikhQRFq5H65SqjFwpZV+Y8zjxph0Y0w68P+AfxtjcusiYKVUw+dWK/2RbgDedCM4pVTj5FYr/QCISAIwBHjn2ENTSjVWUbUgdwSuAv5T3W6QiIwHxgN06NAhuhp3fgEl+UcWhciRFG44dR+JI4rjiCquo3qpw5iPVEOJ4wjVVdgxTaHNOa5WGU1iiaaV/pDrqWE3yBgzA5gBkJmZaaIJcOfc/+HU/dnRFFVKHY0z+8HYD12tMprEEm6lHzuhXA/cWLmQiDQHBgI/cTPA6bG38VXZYOJ9Hvqf04bLzmtLuxbxNbwjqnzlFD2CsnVe95Goo7rrMuS6rTx6dfq51KU6jDu+petVutVKP8AI4P+MMQfdDHDKhJvI+eEAs5Zt4Y9rdvC7L4P0PTuWW85P5uKup+Dz6qU4SjU0Yuopg2dmZprs7CPbxdlXWMqclT/w+vKt/JhXQrsW8dzUpwPXZ3WgVRPtg16puiYiq4wxmbWWa0yJJSQQtPhkw25mLdvC0k37iPF5uCr1dG694ExS27dwN1ClVNgJnVgifb2rgNnLtvLO6m0UlQXpempTOrRKoFWTGFo2iSGpSQwtE2Jo1eTQo2WTGJrEeJEGc5ZCqcbhpEksIfkl5by7ahsfb9jFvsIycg/aj4BV9fLF+Dy0TPDTJMZHjM9DrN9LrM/jPLzE+iOGfR5i/R5ivB68HsErgtcr+DyC1+PB5xE8ntDryGcPfq/g93rweQ+99nkjxnsOn+71HJrm9YgmQNVgnHSJpSrGGApKA+w/WMa+g2UVnnOLysgtLKO4PEhpwLIfznBZwKI0UMX4oFUvJxV8HsHnFfweJwF57SQXSk5+rwe/z4PfSUh+n4eYcELzYIzBMoagZbAMWJbzOnLYeRbsZHbYw0mmXpEKidQTmuYRRAgPh8Yfmh76TOzzG/azCb/GmPB5j9A69ngEj4Anon6PHBoXHvZI+D2mUj3GGRcplLhjvB78vsjh0Hqt+AdBsOcthC7FOfQaQETC0yLnGVpOIpYztNxOLXjEfr83YllFcF4fWqcigmWcepy6LWN/niY83h5nsGMLvd9Tqe7IdSci+L1CQkx0l7RFm1jcvkCuQRERmsX5aRbn58ykJq7UaVmGoPNDDFqGQPjZwrIgYFkELUN50DjPFgHnuTxoEQjaZcuDptKwRbllPweChnLLIhg0h8aF6op4T6jOssCh4fKgRXFxMOK1CX+ZQj90j/PFFRG8EcM+jweDHXdpIEjQQNCyCFqhZ2e5jSEYtJfdilgXliE8PZSwqtlgrObzqngN2JG8Vx293h1bMfdn57ta5wmdWOqCxyN4EPze+o6kcTDmUMIBIv7yH/orX9Ounonc0nL+MgedLTBjHRoO1QmRWxb2VgFS8aJfyzKUOUm33NkSLQscSsTlwUPjgpY5bOsqcguowjQTWh7CWzlUXtaIrZ3QVkdoHqFEHJmUg8bY69AyeJw/AKGtEY8cWn+RWyih+VrWoXUWrjtUX8S4U5vFHevHfBhNLKpORW4VHe37fV49xtTY6NVlSinXaWJRSrlOE4tSynWaWJRSrtPEopRynSYWpZTr6u3KWxHZA2ytl5kfH62BvfUdRB3TZTwxHMkynmmMaVNboXpLLCc6EcmO5tLnxkyX8cRQF8uou0JKKddpYlFKuU4TS92ZUd8BHAe6jCcG15dRj7EopVynWyxKKddpYqkDIrJFRL4QkRwROSE6RRKRl0Vkt4h8GTGulYh8LCLfOM/u9yNxHFWzjFNEZLvzWeaIyOX1GeOxEJEzRGShiKwXkXUiMskZ7/rnqIml7lxojEk/gU5VvoLdfW6kB4B/GWM6A/9yXjdmr3D4MgI87XyW6caYvx/nmNwUAH5pjDkP6APcKSLnUQefoyYWFRVjzGKgcte5VwOvOsOvAsOPa1Auq2YZTxjGmB3GmNXOcAGwAbsfdtc/R00sdcMAn4jIKqe/6hNVW2PMDmd4J9C2PoOpQxNFZK2zq9Sod/dCRCQZ6AGsoA4+R00sdaOfMSYdGIq9uTmgvgOqa8Y+vXginmJ8HugEpAM7gCfrN5xjJyKJwDvA/xhj8iOnufU5amKpA8aY7c7zbmAe0Kt+I6ozu0TkNADneXc9x+M6Y8wuY0zQGGMBL9LIP0sR8WMnldeNMe86o13/HDWxuExEmohI09AwcBnwZc3varTeA251hm8FFtRjLHUi9INzjKARf5Zitzb+V2CDMeapiEmuf456gZzLRKQT9lYK2I2Vv2GMmVqPIblCRN4EBmHfCbsLeAiYD7wFdMC+U/06Y0yjPfhZzTIOwt4NMsAW4GcRxyMaFRHpBywBvgAsZ/SvsY+zuPo5amJRSrlOd4WUUq7TxKKUcp0mFqWU6zSxVENEvCJSKCId3Cxbn0TkbBFx/aCaiFwiIlsiXn8lIv2jKXsU83pJRH59tO9Xx8cJ08WqiBRGvEwASoGg8/pnxpjXj6Q+Y0wQSHS77MnAGNPFjXpEZBzwE2PMoIi6x7lRt6pbJ0xiMcaEf9jOX8RxxphPqisvIj5jTOB4xKZUbU607+NJsyskIg+LyFwReVNECoCfiMj5IrJcRA6IyA4RmeZcmYiI+ETEOPdUICKvOdM/EpECEVkKXUrUAAAgAElEQVQmIh2PtKwzfaiIfC0ieSLynIj8R0TGVBN3NDH+TES+FZH9IjIt4r1eEXlaRPaJyHdUfeduqOxkEZlTadyfReQpZ3iciGxwlmeTszVRXV3bRGSQM5wgIrOd2NYBPSuV/Y2IfOfUu05EhjnjuwN/Avo7u5l7I9btlIj3T3CWfZ+IzI+4grTGdXMk6zkUj4h8IiK5IrJTRO6LmM+DzjrJF5FsETm9qt1OEfks9Dk763OxM59c4Dci0lnsZg1yRWSvs96aR7z/TGcZ9zjTnxWROCfmcyPKnSYiRSKSVN3y1jljzAn3wL6Q6ZJK4x4GyoCrsBNqPJAF9MbecusEfA3c5ZT3YV8Uley8fg27i4RMwA/MBV47irKnAAXYd5T6gXuAcmBMNcsSTYwLgOZAMvbduZc40+8C1gHtgSRgMc7tIFXMpxNQCDSJqHs3kOm8vsopI8BFQDGQ6ky7BNgSUdc2YJAz/ASwCGgJnAmsr1T2OuA05zO50YmhrTNtHLCoUpyvAVOc4cucGNOBOOAvwKfRrJsjXM/NsS+YmwTEAs2AXs60/wesATo7y5AOtALOrryugc9Cn7OzbAHg54AX+/t4DnAxEON8T/4DPBGxPF8667OJU76vM20GMDViPr8E5tXrb7C+k0CdLFT1ieXTWt53L/B2pS9mZLKYHlF2GPDlUZS9DVgSMU2wb26rMrFEGWOfiOnvAvc6w4uxdwlD0y6v/GWvVPdy4EZneCjwVQ1lPwDudIZrSizfR34WwB2RZauo90vgCme4tsTyKvC/EdOaYR9Xa1/bujnC9XwzsLKacptC8VYaH01i+a6WGK4NzRfoj33nsbeKcn2BzRy64DUHGOn27+pIHifNrpDjh8gXItJVRD50Nm3zgd9jX85dnZ0Rw0XUfMC2urKnR8Zh7G/CtuoqiTLGqOZF7R3EvQHc4Azf6LwOxXGliKxwNtMPYG8t1LSuQk6rKQYRGSMia5zN+QNA1yjrBXv5wvUZ+07d/dhtjIRE9ZnVsp7PwE4gValpWm0qfx9PFZG3xG6xLh+74anIGLYY+0RBBcaY/2Bv/fQTkRTsS/M/PMqYXHGyJZbKp1pfwP4LebYxphnwW+wtiLq0A/svKhC+Maxd9cWPKcYd2F/IkNpOh78FXCIiocZ/3nBijAf+BvwRezelBfB/Ucaxs7oYxL6v6nns3YEkp96NEfXWdmr8R+zdq1B9TbF3ubZHEVdlNa3nH4CzqnlfddMOOjElRIw7tVKZysv3KPbZzO5ODGMqxXCmiHiriWMW8BPsrau3jDGl1ZQ7Lk62xFJZUyAPOOgc/PrZcZjnB0CGiFwlIj7s/faauqw8lhjfAv5HRNo5B/Lur6mwMWYn9ub6K9i7Qd84k2Kx9/v3AEERuRL7WEC0MfxaRFqIfZ3PXRHTErF/XHuwc+zt2FssIbuA9pEHUSt5E/ipiKSKSCx24ltijKl2C7AGNa3n94AOInKXiMSKSDMRCTWf8BLwsIicJbZ0EWmFnVB3Yp8k8Ird4NeZ1KwpdkLKE5EzsHfHQpYB+4D/FfuAeLyI9I2YPht71+lG7CRTr072xPJL7NvEC7D/Ys2t6xkaY3YBo4GnsL8oZwH/xf5L5XaMz2O3YfoFsBJ7q6M2b2AfMwnvBhljDgC/wL5rOxf7C/xBlDE8hL3ltAX4iIgvvTFmLfAc8LlTpgv2nbYhHwPfYLcXErlLE3r/P7B3WeY57+8A3BRlXJVVu56NMXnApcA12Mnua2CgM/lx7Lu8/wXkYx9IjXN2cW/Hvnt4L/Yxl8hlq8pD2O295GEns3ciYggAVwLnYm+9fI/9OYSmb8H+nEuNMUuPcNldp3c31zNn0/ZH4FpjzJL6jkc1XiIyC/uA8JT6juWEuUCuMRGRIdhnYIqxT1eWY//VVuqoOMerrga613csoLtC9aUf8B32sYXBwIj6PtimGi8R+SP2tTT/a4z5vr7jAd0VUkrVAd1iUUq5ThOLUsp19XbwtnXr1iY5Obm+Zq+UOgqrVq3aa4yp6boroB4TS3JyMtnZJ0R/6UqdNESktttCAN0VUkrVAU0sSinXaWJRSrlOE4tSynWaWJRSrtN7hRqYgBWgOFBMUXmR/RwoOmzYMhbNY5vTPLY5zWKahYfjvHHYzbscuXKrnJJAyWHzKgoUheOJHC4OFBOwAiTFJ9E6vjVt4tvQOr41reNb0yq+FX5PdS0dHLtyq5y80jz2l+znQOkBisqLaBnXMjz/GG/M0dcdLGdP8R52F+1mT/Ee8krzKLfKKQ+WEzAByoPl9uvQI+K1ZSxaxrUMr4s2CW3Cw81imtX62RhjyCvNY0/xHvYW7w0/9hTvIb80H4948IgHEcGD/ewVb4VxoWGvePF7/Pg8PnweH17xhof9Hj9ejxef2K+T4pPo2bZnjbEdKU0sUdhbvJfsndkUB4pJPyWd5GbJR/0DDikNlrJq1yqWbl/K8h3L2VW0i+JAMaXBo79lKMYTE04yzWOb0zymOYkxieFkVRwopiRQYj8HSygut8cVB+0kES2PeEjwJSAiFJQVHDZdEFrGtSQpPin8w0qKTyLWG4sgiAiC2D8C53Vo2CP2RnRZsIz9pfsrJJDQc2F54WHzjNQsptmhRJfQmtZxrcPDSXFJlARKKiSP3UW72VO0hz3Fe8gtqb0vdEHwe/z4vX772XmICLkluRQHig97j98TQ/OYVjT3J5Hob0WirxXlVjn5ZbkUlOdSUL6fwsB+LA7/HHwSS4zYDd8ZLMDY/4yFwQBWaAwGyxlvOWVrl9ykO+9f+0btBY+AJpYq7C7aTfbObFbuWkn2zmy25G+pML1VXCsyTsmgZ9ueZLTNoEvLLng91TXsZTPG8F3ed/xn+39Y+uNSsndlUxosxe/xk3FKBumnpJPgSyDeH0+CL4EEfwLxvkPDCT7ntT8BDx7yy/LJK80jryyPfcUH2F2Yy56i/ewrPkBeaR57C/LYGthEcfAgXnz4JBa/Jw6fJw6/NCfGcwqJvjj8MXHEeGKJ8cTh98Thl1h8EodX4vARi4c4vMTiIRaPsZ+N8WEMBC1DuVVOsXWA4uB++9k6QLG1n9LgAfLyD7A7bxel5itKTR6mih9NTTwmFo9pgsckgpUAwbaYQCe8wXiC5QkEyhMoL4/HCsbg9RfhjynE5y/koL+QIl8+W727MZ5vsTz5GCmv4kMRfDTDZ1rgMy3wWim0tJrjsZojweZ4rGaYYAJW0EvQeAkGPFjGS9ASgpah2DIUWoag8whYFpYBPKWILx+PrwDxFSC+fMp8BRT58tnpK0B8X+Hx5WOMFxNo6jzOwAp0wwQSMYFmmEAiVqAZJtgUMbF4nT9kxvku2c+1rUELxHKeg4gEndf2c+i1t537jflHlVic2/yfxW5N/CVjzCNVlBkEPIPd8vxeY8zAymUaqp0Hd7Jy50pW7VrFyp0r+b7AvkE00Z9IRtsMRnYeSdapWST4E/jvrv+yevdqsndl88n3drdF8b4EurZI5aym3TmzSQptYs4mEPSSX5rPtwWr2VS4mu8OrqYgsBeAFv52nJNwKe1j0zjF3w0PMQQOGEosQ0HQsn+wQUMgaBGwjP0IllMeLCNg7aekPEhecYC8ojIOFAcpKkvA7qOtfTVLeCws7NYd7L/CHgGvR/CI4PUIXhFEwONJwCNNENo5WyDgcZ5jRIh13gcGjwBi8IggWHhEQAwi9jQRg9cTQ7w3lhifx374PYeGfR5ivB5inWGvRygPWpQFnEfQorTcotQZVxoIUhI4SFFwPyXmAGJiiaEFfmmGT3z2cjgPj08ilg9nvAevYD977GdfxHvCDxFifIfisoe9FcbFej3E+j3EeL34fYLPqcvnFfxeZ9jjwed1xnk8eDy170IZUzHpHJoWMRwxJXL8MW58V6nWu5udhoi+xm5Baxt2S2Q3GGPWR5RpASwFhhhjvheRU4wxu2uqNzMz0xzPK28DwQBf79vO2p2b+WrfVrbm/cCOou3sC3xNsROqlwSamnNIMOeQEDwHX6A9AYvwD700EKS4LMjB0gBFZUGCngN4Ezbbj/gteON2AWAsH1Z5KzwxexAxmGAcgYNnEzx4DoHCzphAy8PiC3+5PB68XvvL5Xe+XIe+fPa4WJ+H5vExtEjw0zzeT4t4vz2cEFPhdYv4GJrG+RCxv0hBY7CcL2HQsoctA5YzHDQG4dCPxOu1nz0e7NceOeZdQNW4icgqY0xmbeWi2WLpBXxrjPnOqXgOdoMy6yPK3Ai8G2oLorak4iZjTPhgY0FpITnbf2TD3i1sPrCNHUXbyS3dSZG1h4An1970C79PkGAzfIEOJJT3JTbQmXjOwO/1On+Z7B9WnN8e9nnsvzpNYr0kxPgOPcecT0KsjyYxPoynkO3FG9hc+AU7irfStcUwMtv24bxW3fD7fPg8ngo/2PBfSaHOf7Ai4KnzdsKVskWTWNpRsZuCbdgdO0U6B/CLyCLsBoGfNca40qDvy1++zNf7v+Zg+UGKyos4WH4wPFwUsB+WqeYgVTCBGFrTwptMm7jzad+0HWe17MB5pySTdmpHkpo0cSPESjpjdyOk1MnLrYO3PuyuMy/G7qFtmYgsN8Z8HVnIaal8PECHDrX1RGFbt3cdG3I3kOBLoIm/CS3jWtK+afvwa7/EsXpLESs3FxHvTWBE2tn0OfMs0k7txKlNW7i0eEqpIxFNYtlOxX5h2nN4vy3bgH3GmIPY3ScsBtKwj82EGWNmYLdiTmZmZlRN1z056Mlqp/1z3U6mvLeOHXkl3Ni7A/cP7krzhLq7fkIpFZ1oEstKoLPYnZpvB67HPqYSaQHwJ6efnBjsXaWn3Qw00rb9RUx5bz2fbNhF11Ob8qcbM+h55uEHRJVS9aPWxGKMCYjIXcA/sU83v2yMWSciE5zp040xG0TkH8Ba7POTLxljvnQ72PKgxcufbeaZT+x+tCZffi5j+ibj9+qdCUo1JPXWmPaRnm7O3pLL5Hlf8tWuAi49ry1ThnWjXYv4OoxQKVWZm6eb69WBojIe+Wgjc1b+wOnN45hxc08u61a5C1ylVEPSoBOLMYabXlrBxp0FjB/QiUkXd6ZJbIMOWSlFA08sIsLkK86lZUIM557WrL7DUUpFqUEnFoALzmpd3yEopY6Qnk5RSrlOE4tSynWaWJRSrtPEopRynSYWpZTrNLEopVyniUUp5TpNLEop12liUUq5ThOLUsp1mliUUq7TxKKUcp0mFqWU6zSxKKVcF1ViEZEhIvKViHwrIg9UMX2QiOSJSI7z+K37oSqlGota22Nxulj9MxFdrIrIe5FdrDqWGGOurIMYlVKNTDRbLOEuVo0xZUCoi1WllKpSNImlqi5W21VR7gIRWSsiH4lIN1eiU0o1Sm41Tbka6GCMKRSRy4H52J0YV3A0XawqpRqfaLZYau1i1RiTb4wpdIb/jt1B/GGN1RpjZhhjMo0xmW3atDmGsJVSDVk0iSXcxaqIxGB3sfpeZAEROVVExBnu5dS7z+1glVKNgytdrALXAj8XkQBQDFxv6quLRaVUvWs0XawqpepftF2s6pW3SinXaWJRSrlOE4tSynWaWJRSrtPEopRynSYWpZTrNLEopVyniUUp5TpNLEop17l1d/NJq7y8nG3btlFSUlLfoSjlmri4ONq3b4/f7z+q92tiOUbbtm2jadOmJCcn49yHqVSjZoxh3759bNu2jY4dOx5VHbordIxKSkpISkrSpKJOGCJCUlLSMW2Fa2JxgSYVdaI51u+0JhallOs0sSjXLFq0iKVLl9ZYZsyYMfztb387bHx2djZ33313XYWmjjM9eKtcs2jRIhITE7nggguO+L2ZmZlkZtbazEdUgsEgXq/XlbrcFAgE8PlOjp+cbrGcAGbNmkVqaippaWncfPPNvP/++/Tu3ZsePXpwySWXsGvXLgCmTJnCzTffzPnnn0/nzp158cUXAdixYwcDBgwgPT2dlJQUlixZAkBiYiKTJ08mLS2NPn36hOvZs2cP11xzDVlZWWRlZfGf//yHLVu2MH36dJ5++mnS09PDdVTlk08+ITMzk3POOYcPPvgAsJPSlVdeGY7ztttuY9CgQXTq1Ilp06aF3zt8+HB69uxJt27dmDFjRnh8YmIiv/zlL0lLS2Pq1KkMHz48PO3jjz9mxIgR1cbz85//nMzMTLp168ZDDz0UHr9y5UouuOAC0tLS6NWrFwUFBQSDQe69915SUlJITU3lueeeAyA5OZm9e/cC9tbXoEGDKqzzvn37cvPNN7Nlyxb69+9PRkYGGRkZFbbwHn30Ubp3705aWhoPPPAAmzZtIiMjIzz9m2++qfC6IYsqfYrIEOBZ7KYpXzLGPFJNuSxgGXbTlIdv757gfvf+Otb/mO9qneed3oyHrqq+N5V169bx8MMPs3TpUlq3bk1ubi4iwvLlyxERXnrpJR577DGefPJJANauXcvy5cs5ePAgPXr04IorruDNN99k8ODBTJ48mWAwSFFREQAHDx6kT58+TJ06lfvuu48XX3yR3/zmN0yaNIlf/OIX9OvXj++//57BgwezYcMGJkyYQGJiIvfee2+Ny7RlyxY+//xzNm3axIUXXsi33357WJmNGzeycOFCCgoK6NKlCz//+c/x+/28/PLLtGrViuLiYrKysrjmmmtISkri4MGD9O7dmyeffBJjDOeeey579uyhTZs2zJw5k9tuu63aeKZOnUqrVq0IBoNcfPHFrF27lq5duzJ69Gjmzp1LVlYW+fn5xMfHM2PGDLZs2UJOTg4+n4/c3NxaP8P169fz2WefER8fT1FRER9//DFxcXF888033HDDDWRnZ/PRRx+xYMECVqxYQUJCArm5ubRq1YrmzZuTk5NDeno6M2fOZOzYsbXOryFwrSdEp9yjwP/VRaCqap9++imjRo2idWu7U4RWrVrxxRdfMHr0aHbs2EFZWVmFaxGuvvpq4uPjiY+P58ILL+Tzzz8nKyuL2267jfLycoYPH056ejoAMTEx4a2Inj178vHHHwP2Fsf69Yc+/vz8fAoLC6OO+brrrsPj8dC5c2c6derExo0bDytzxRVXEBsbS2xsLKeccgq7du2iffv2TJs2jXnz5gHwww8/8M0335CUlITX6+Waa64B7DMaN998M6+99hpjx45l2bJlzJo1q9p43nrrLWbMmEEgEGDHjh2sX78eEeG0004jKysLgGbNmoWXfcKECeFdmlatWtW6vMOGDSM+Ph6wL6i86667yMnJwev18vXXX4frHTt2LAkJCRXqHTduHDNnzuSpp55i7ty5fP7557Wv4AYgmi2WcE+IACIS6gmxcherE4F3gCxXI2xEatqyOJ4mTpzIPffcw7Bhw1i0aBFTpkwJT6t8GlFEGDBgAIsXL+bDDz9kzJgx3HPPPdxyyy34/f5wea/XSyAQAMCyLJYvX05cXNxRxVdVDJXFxsaGh0PzXrRoEZ988gnLli0jISGBQYMGha+1iIuLq3BcZezYsVx11VXExcUxatSoao9tbN68mSeeeIKVK1fSsmVLxowZc1TXb/h8PizLAjjs/U2aNAkPP/3007Rt25Y1a9ZgWVat6/Caa67hd7/7HRdddBE9e/YkKSnpiGOrD670hCgi7YARwPPuhaaicdFFF/H222+zb5/d20pubi55eXm0a2d/RK+++mqF8gsWLKCkpIR9+/axaNEisrKy2Lp1K23btuX2229n3LhxrF69usZ5XnbZZeFjCwA5OTkANG3alIKCglpjfvvtt7Esi02bNvHdd9/RpUuXqJY1Ly+Pli1bkpCQwMaNG1m+fHm1ZU8//XROP/10Hn744Rp3H/Lz82nSpAnNmzdn165dfPTRRwB06dKFHTt2sHLlSgAKCgoIBAJceumlvPDCC+EkG9oVSk5OZtWqVQC88847NS7DaaedhsfjYfbs2QSDQQAuvfRSZs6cGd4NDdUbFxfH4MGD+fnPf95odoPAvYO3zwD3G2OsmgqJyHgRyRaR7D179rg065Nbt27dmDx5MgMHDiQtLY177rmHKVOmMGrUKHr27BneRQpJTU3lwgsvpE+fPjz44IOcfvrpLFq0iLS0NHr06MHcuXOZNGlSjfOcNm0a2dnZpKamct555zF9+nQArrrqKubNm1frwdsOHTrQq1cvhg4dyvTp06Pe8hkyZAiBQIBzzz2XBx54gD59+tRY/qabbuKMM87g3HPPrbZMaLm7du3KjTfeSN++fQF7N3Du3LlMnDiRtLQ0Lr30UkpKShg3bhwdOnQIHyx/4403AHjooYeYNGkSmZmZNZ6RuuOOO3j11VdJS0tj48aN4a2ZIUOGMGzYMDIzM0lPT+eJJ56osBwej4fLLrssqvXUENTa/YeInA9MMcYMdl7/PwBjzB8jymwGQtuzrYEiYLwxZn519Z4o3X9s2LChxi9uQzJlypSoDq6eKO666y569OjBT3/60/oO5Zg88cQT5OXl8Yc//OG4zreq73a03X9Ec4wl3BMidteq1wM3RhYwxoSPDorIK8AHNSUVpepaz549adKkSfhsWGM1YsQINm3axKefflrfoRwRt3pCVI1A5EHcujZ16lTefvvtCuNGjRrF5MmTj8v8Q8c7IvXu3ZvS0tIK42bPnk337t2PS0xHI3QGrLHRnhCPUWPaFVLqSBzLrpBeeauUcp0mFqWU6zSxKKVcp4lFKeU6TSwnmClTplS4uKoh1l1dPT/++CPXXnvtMdev6p8mFlVnQpe9R+v000+vshGooxG6VL6hOdJ10lhpYjkBTJ06lXPOOYd+/frx1VdfAbBp0yaGDBlCz5496d+/f/gO4l27djFixAjS0tJIS0sLtwfy1FNPkZKSQkpKCs8880yNdddU/5gxY5gwYQK9e/fmvvvuqzbmNWvWHNYuzJYtW0hJSQHglVdeYeTIkQwZMoTOnTtXqKu69lOSk5O5//77ycjI4JFHHjmitkx+//vfk5WVRUpKCuPHjyd0Gca3337LJZdcQlpaGhkZGWzatAk4vO0UgEGDBhG6hGLv3r0kJyeHl2XYsGFcdNFFXHzxxRQWFnLxxReTkZFB9+7dWbBgQTiOym3rFBQU0LFjR8rLywH73qbI1w3VydGc1fHy0QOw8wt36zy1OwytsvkbwL4QbM6cOeTk5BAIBMjIyKBnz56MHz+e6dOn07lzZ1asWMEdd9zBp59+yt13383AgQOZN28ewWCQwsJCVq1axcyZM1mxYgXGGHr37s3AgQOxLKvKuoFq6we7S5SlS5fWeM9MVe3CVJaTk8N///tfYmNj6dKlCxMnTuSMM86osv2U1NRUAJKSksI3UX7yySdRt2Vy11138dvf/haAm2++mQ8++ICrrrqKm266iQceeIARI0ZQUlKCZVlVtp1Sm9WrV7N27VpatWpFIBBg3rx5NGvWjL1799KnTx+GDRvG+vXrD2tbp2nTpgwaNIgPP/yQ4cOHM2fOHEaOHHnU/f0cL5pYGrklS5YwYsSIcDsew4YNo6SkhKVLlzJq1KhwudAVp59++mm4bRKv10vz5s357LPPGDFiRPiGuJEjR7JkyRIsyzqsboDCwsJq6wf7Ctvamoasql2YUDswIRdffDHNmzcH4LzzzmPr1q2cccYZVbafEkoso0ePDr//SNoyWbhwIY899hhFRUXk5ubSrVs3Bg0axPbt28Otz4Vulqyu7ZSaXHrppeFyxhh+/etfs3jxYjweD9u3b2fXrl1Vtq0TWo7HHnuM4cOHM3PmzPAWXkOmicVNNWxZHE+WZdGiRYtwcwbHu/7I9keqc7RtstTWfkrkvKNty6SkpIQ77riD7OxszjjjDKZMmVKnbbK8/vrr7Nmzh1WrVuH3+0lOTq5xfn379mXLli0sWrSIYDAY3l1syPQYSyM3YMAA5s+fT3FxMQUFBbz//vskJCTQsWPH8L06xhjWrFkD2FsBzz9vN5sTDAbJy8ujf//+zJ8/n6KiIg4ePMi8efPo379/lXWD3ZpadfVHq6p2YaJRXfspVYm2LZPQj7p169YUFhaGDyA3bdqU9u3bM3++fT9taWkpRUVF1badEtkmS00HofPy8jjllFPw+/0sXLiQrVu3AlW3rRNyyy23cOONNzaaNlk0sTRyGRkZjB49mrS0NIYOHRr+gb7++uv89a9/JS0tjW7duoUPED777LMsXLiQ7t2707NnT9avX09GRgZjxoyhV69e9O7dm3HjxtGjR49q666p/mhV1S5MNKprP6U60bRl0qJFC26//XZSUlIYPHhwheWcPXs206ZNIzU1lQsuuICdO3dW23bKvffey/PPP0+PHj3CDWtXF1N2djbdu3dn1qxZdO3aFai6bZ3I9+zfv58bbrghqvVU3/QmxGOkNyE2bPXVlonb/va3v7FgwQJmz5593OZZ1+2xKNUoNda2TCqbOHEiH330EX//+9/rO5SoaWJRdWbmzJk8++yzFcb17duXP//5z8dl/lW1ZTJixAg2b95cYdyjjz7K4MGDj0tMRyOyfeHGQhOLqjNjx45tcAcbG2vDSY2NHrxVSrlOE4tSynVRJRYRGSIiX4nItyLyQBXTrxaRtSKS43Tv0c/9UJVSjYVbXaz+C3jPGGNEJBV4C+haFwErpRq+aLZYwl2sGmPKgFAXq2HGmEJz6IKYJkD9XByjapWYmFjfIURt/vz5FfqIrkrkHcWR3nvvPR55pGHcYnEycqWLVQARGSEiG4EPgdvcCU+dzKJJLNUZNmxYuDmDY2GMCd//09A05LZdXDvdbIyZB8wTkQHAH4BLKpcRkfHAeLC72TzRPPr5o2zM3ehqnV1bdeX+XvdXO/2BBx7gjDPO4M477wTs1tl8Ph8LFy5k//79lJeX8/DDD3P11VdXW0ekRx99lNdeew2Px8PQoUN55JFHePHFF5kxYwZlZWWcffbZzJ49m4SEBMaMGUNcXBzZ2dnk5+fz1FNPcUHWR/EAABX6SURBVOWVV7Ju3TrGjh1LWVkZlmXxzjvv4Pf7GTp0KP369WPp0qW0a9eOBQsWEB8fz6ZNm7jzzjvZs2cPCQkJvPjii+Tm5vLee+/x73//m4cffph33nmHs846q8qYZ8+ezbhx4wgEArz88sv06tWLV155hezsbP70pz8xZswYmjVrRnZ2Njt37uSxxx7j2muvpbCwkKuvvvqw9bRlyxYGDx5M7969WbVqFddddx379+8Pt1Pz4osvsn79ep5++ukq4xk+fDg//PADJSUlTJo0ifHjxwPwj3/8g1//+tcEg0Fat27Nv/71LwoLC5k4cSLZ2dmICA899BDXXHMNiYmJFBYWAvZVtx988AGvvPJKeJ3/97//pW/fvlx//fVMmjSJkpIS4uPjmTlzJl26dCEYDHL//ffzj3/8A4/Hw+233063bt2YNm1a+N6njz/+mL/85S91cgo+msSyHTgj4nV7Z1yVjDGLRaSTiLQ2xuytNG0GMAPsS/qPIl5VyejRo/mf//n/7d15cFRVvsDx7687SwcXDMhAEBF4D9kck7yAQZhBwMJxGzRgyCCyiDCCGHDUUkaYpxaDwxtBa6ypN4AL8FzD4jLlSE0RRioyg5pIBUSIssi4hRhRICxCL+f9cW9fOkl30pHbCYm/T1XXvX1u97nnbr/uvn3v79zjBJbVq1fz97//ndmzZ9fL9xHtDuJIsfKMjBkzhunTpwMwf/58nn32WQoLCwErOdP777/P3r17GTFiBHv27GHp0qXMmTOHCRMmcOrUKYLBIFVVVezevZuXX36Zp59+mnHjxrFu3Tpuu+22mLldRo8ezY033thousrjx49TXl5OSUkJU6dOZceOHfVeU1lZyebNm6moqGD06NHccsst+Hy+qHlRwEoMtWrVKgYPHszRo0fJzMzk8ccfJzk5mRUrVrBs2bKY7Xnuuefo0KEDJ06cYNCgQYwdO5ZQKMT06dMpKSmhZ8+ezrpdsGAB7du358MPrTw+3333XYPLCrXz3Rw5coR33nmHpKQkiouLeeihh1i3bh3Lly9n//79lJeXk5SUxLfffkt6ejp33XUX1dXVdOrUiRUrVjB1amJ+XLjSxaqI/Cew1z55+19AKnDQ7cae7Rr6ZpEo2dnZfP3113z11VdUV1eTnp5Oly5d+M1vflMv30eXLl0arCtWnpEdO3Ywf/58Dh06xNGjR2tdpTpu3Dg8Hg+9e/emV69eVFRUcOWVV7Jw4UK++OILxowZQ+/evQHo2bOnk3MlJyeH/fv3N5rbJR7hG/OGDRvGkSNHOHToUL3X3HzzzXg8Hvr3709VVRUQOy8KwCWXXOJ0On/uuecycuRI3nzzTfr164ff72+w98SnnnrK+Rbw+eefs3v3bqqrqxk2bBg9e/astW6Li4t55ZVXnPemp6c3uryR+W4OHz7M5MmT2b17NyLiZJYrLi5mxowZJCUl1ZrfxIkTeeGFF7j99tvZsmWLk5vHbW51sToWmCQifuAEUGBa6u7GH6H8/HzWrl3LgQMHKCgoaHK+j8ZMmTKF119/nczMTFauXMmmTZucadHyqtx6663k5ubyt7/9jeuvv55ly5bRq1evevlVTpw44UrumKbmdgnvmg2tp7o5ZaZNm8Zjjz1G3759G7yaeNOmTRQXF7NlyxbatWvH8OHDf9C6j1yGhnK7/O53v2PEiBG89tpr7N+/n+HDhzdY7+23384vf/lLfD4f+fn5TuBxW1zXsRhj3jLGXGqM+Q9jzEK7bGm432ZjzP8YYwYYY7KMMVcaYzYnpLUqqoKCAl555RXWrl1Lfn5+zHwfjYmVZ6SmpoaMjAz8fj8vvvhirfesWbOGUCjE3r172bdvH3369GHfvn306tWL2bNnc9NNN7F9+/aY82wot8t5551HTU1No+0uKioCYPPmzbRv397JOteYpqyn3NxcPv/8c1566aUGUxccPnyY9PR02rVrR0VFBe+++y4AgwcPpqSkxLlPKbxuR40aVeveqfBPoc6dO7Nr1y5CoVCD50AOHz7MRRdZ/6WsXLnSKR81ahTLli1zTvCG59e1a1e6du3K73//+4TebqFX3rYBAwYMoKamhosuuoiMjIyY+T4aEyvPyIIFC8jNzWXo0KH16urevTtXXHEF1113HUuXLsXn87F69Wouu+wysrKy2LFjB5MmTWpwvrFyu/zqV7/i8ccfJzs720liHY3P5yM7O5sZM2bw7LPPxrWsEDsvSizjxo1j6NChDf5cufbaawkEAvTr14+5c+c6P6c6derE8uXLGTNmDJmZmU4Kzfnz5/Pdd99x2WWXkZmZydtvvw3AokWLuPHGGxkyZAgZGRkx5/fAAw/w29/+luzs7Fr/Ek2bNo3u3bs7iblfeumlWst98cUXJzbdhzGmRR45OTmmLdi5c2dLN6HFTJ482axZs6alm9FsbrjhBlNcXNzSzThjs2bNMs8880yjr4u2bwNlJo7jW7+xKNWIQ4cOcemll5KWlsbVV1/d0s05Izk5OWzfvp3bbrstofPRtAk/Qh9++CETJ06sVZaamsp7773XpHoif9Mn2qxZs/jnP/9Zq2zOnDnNkpbhggsu4JNPPqlVdvDgwahBZuPGjTGTdp8Nwjl5E00Dy4/QT3/604Rl8E+U5koOFa+OHTu2unXYnPSnkFLKdRpYlFKu08CilHKdBhallOs0sPzINCUfy5QpUxrs0e9MuVV/rHrKysqYPXv2Gdevmk7/FVKtRiAQaNK9LQMHDmTgwEb71opLMBhstKP7ltDUddJczr4WtWIHHnuMk7vczceS2q8vXR56KOZ0N/OxGGMoLCxkw4YNXHzxxaSkpDjTPvjgA+69916OHj3KhRdeyMqVK8nIyGDPnj3MmDGD6upqvF4va9asoVevXjzwwAOsX78eEWH+/PkUFBT8oPqHDx9OVlYWmzdvZvz48dx3331R215cXMyiRYtq5YXZtGkTixcv5s033+SRRx7hs88+Y9++fXz22Wfcc889zreZWPlTzj33XO68806Ki4sZO3YsW7dujTuXycyZMyktLeXEiRPccsstPProowCUlpYyZ84cjh07RmpqKhs3bqRdu3b1cqcUFhbSo0cPysrKuPDCCykrK+P+++9n06ZNPPLII869Wd27d+cPf/gDEydO5NixYwD8+c9/ZsiQIUD9/DrTp08nPz+frVu3AlZ6iIKCAue5WzSwtHJu5mN57bXX+Pjjj9m5cydVVVX079+fqVOn4vf7KSws5I033qBTp04UFRUxb948nnvuOSZMmMDcuXPJy8vj+++/JxQK8eqrr1JeXs62bdv45ptvGDRoEMOGDWPLli1Nrh/g1KlTUdNPRoqWF6auiooK3n77bWpqaujTpw8zZ84kOTk5av6Ujh07cuzYMXJzc1myZAnGGPr16xd3LpOFCxfSoUMHgsEgV199Ndu3b6dv374UFBRQVFTEoEGDOHLkCGlpaVFzpzRm586dbN68mbS0NI4fP86GDRvw+Xzs3r2b8ePHU1ZWFjW/TocOHWjfvj3l5eVkZWWxYsWKhFxkqIHFRQ19s0gUN/OxlJSUMH78eLxeL127dmXkyJEAfPzxx+zYsYNRo0YB1s+CjIwMampq+PLLL8nLywOsmwEB59uF1+ulc+fOXHXVVZSWlja5/rDwDXsNiZYXpq4bbriB1NRUUlNT+clPfkJVVRXdunWLmj+lY8eOeL1exo4dC1hpDJqSy2T16tUsX76cQCBAZWUlO3fuRETIyMhwOp0///zzgdi5UxoyevRo0tLSAPD7/dx9992Ul5fj9Xqdq4Rj5deZNm0aK1as4IknnqCoqIj333+/0fk1lQaWNiDR+ViMMQwYMIAtW7bUKo8npcGZ1B9WNzdKNE3NyeL1egkEAg3mT/H5fLXOq8Sby+TTTz9l8eLFlJaWkp6ezpQpU37Q+k9KSnLy7TaUk+XJJ5+kc+fObNu2jVAo5AT4WMaOHcujjz7KyJEjycnJScgtCPqvUBvgVj6WYcOGUVRURDAYpLKy0rmFv0+fPlRXVzsHvt/v56OPPuK8886jW7duznmHkydPcvz4cX7+85879VRXV1NSUsIVV1zR5PqbIlpemHjEyp8STby5TI4cOcI555xD+/btqaqqYv369YC1nJWVlZSWlgJWYA4EAjFzp/To0cO5t2fdunUNLkNGRgYej4fnn3+eYDAIxM6v4/P5+MUvfsHMmTMTdq+VBpY2wK18LHl5efTu3Zv+/fszadIkrrzySgBSUlJYu3YtDz74IJmZmWRlZfGvf/0LsBJZP/XUU1x++eUMGTKEAwcOkJeX5+QBGTlyJH/84x/p0qXLD6o/XtHywsQjVv6UWOLJZZKZmUl2djZ9+/bl1ltvZejQoYC1nEVFRRQWFpKZmcmoUaP4/vvvY+ZOefjhh5kzZw4DBw5s8B+pu+66i1WrVpGZmUlFRYXzbSZWfp3wcng8Hq655pq41lNTiYkjg6SIXAv8CSs15TPGmEV1pk8AHgQEqAFmGmO2NVTnwIEDTWMn5FqDXbt2JTZhjjqr3H333WRnZ3PHHXe0dFPOyOLFizl8+DALFiyI+Zpo+7aIfGCMafQ/fLd6QvwUuMoY852IXIeViT+3sbqVak1ycnI455xzWLJkSUs35Yzk5eWxd+9e/vGPfyRsHvGcvHV6QgQQkXBPiE5gMcZEfm99F6uLEHWWcisfS3NbuHChkxs3LD8/n3nz5jXL/KPlMsnNza3Xq8Dzzz/fYBb/lpaIfoTqiiewROsJsaFvI3cA68+kUSqxWmM+FoB58+Y1WxCJ19kejFuKq383i8gIrMDysxjT22RPiMaYRi8+U6o1iefca0Pi+Vcorp4QReRy4BngJmNM1M7KjDHLjTEDjTEDO3Xq9EPae9bx+XwcPHjwjDeEUmcLYwwHDx6M+5+1aNzqCbE78Cow0RjzSf0q2q5u3brxxRdfUF1d3dJNUco1Pp+Pbt1++KlSt3pC/G+gI/C/9k+CQDx/SbUFycnJTreZSilLXNexJEJbuY5FqR+TeK9j0StvlVKu08CilHKdBhallOs0sCilXKeBRSnlOg0sSinXaWBRSrlOA4tSynUaWJRSrtPAopRynQYWpZTrNLAopVyngUUp5ToNLEop12lgUUq5TgOLUsp1GliUUq7TwKKUcl1cgUVErhWRj0Vkj4jMjTK9r4hsEZGTInK/+81USrUmbnWx+i0wG7g5Ia1USrUq8XxjcbpYNcacAsJdrDqMMV8bY0oBfwLaqJRqZeIJLNG6WL0oMc1RSrUFzXryVkR+LSJlIlKmHXwp1Xa51sVqPNpiF6tKqfriCSxOF6sikoLVxepfE9sspVRr5koXqyLSBSgDzgdCInIP0N8YcySBbVdKnaXi6RQeY8xbwFt1ypZGjB/A+omklFJ65a1Syn0aWJRSrtPAopRynQYWpZTrNLAopVyngUUp5ToNLEop12lgUUq5TgOLUsp1GliUUq7TwKKUcp0GFqWU6zSwKKVcp4FFKeW6uNImqNbLhEIQCGD8fusRsB74A9a4/5Q1PWBNJxjABAMQDGICAQgFIRRwxq1pIUzQ77zG+AMQ8FvjgQAmYL/XeR7ABINgAAyIIAAi1nMEq8AmAsaACUEo5IybcJkxdnnIWj5jrDaFgpiA9R4TtB7hckIGEwxigiFrtl4P4vUgHkG8Huu5R04/9wjitdsVMlZ9oRAEjT0MYexyQpHj9vTIodM+47yWkLHn4UG84aHHbosgSXZ77CEi9noNOcOQ337uP10esoenV2vEupXT61ciytJ6d6PbC2+6ut+d/YGlcjvmxCFr5Z0KEAoEMCetAyV00o/xBwid8mPCD78fc+oUxn/SOmj8p04fQH5raO309kEWsnfeUDBi3Fg7Y3gHDoUwJlxu7B3dYAzWDmLsncdgjYeMffwYe2MaeyOffm6NG8B6nbWzYe2gIewdkvrl4WPLnof12tPHIRHHntUeibJSm5kYxAN2ZAHk9GhEaWTZ6fdGxJxa6zKyDMRjl0vEuMdYB5DY28ITEbMMmJDY60ucdVa3DCTivXZ9dr3isZctyjwiyz1yum2IQbw4R54JgTklYM8v5LRF7DaIs13FC+K11qXHa9Xj8djDNGMtrze8/HJ65SCYWutc7NVtvSb1ghNN2JjxOesDy7+nTOb4v793t1J7R3cO/PBGJ2JcpN64E+kbeI04r8XZmM5GrTsePuhDYn16ecT5pBSP2J+qkeX2p5cnXO5xPnnxehGPPe7xIF6v/SnsQZK8p1/jtaZJUvi9XiTJa70nyWu9zxMeepzn1nu94ElCkpLA43HKJCkZSfYi3iSw65DkJPt5ktXOuis5YnD6eZ2NYB0ltR8eT/0y8dSpI9Z4uPoYdTgPqT3etJ2rgXlH3dEi5ncWfAi4JK7AIiLXAn/CSk35jDFmUZ3pYk+/HjgOTDHGbHWjgeffMpF2lV8hKcl4UpKsHTYlCU9KCpLixZOcjNjlp6f7IMWHpKYhKamQ0g5JSUNS06zypFTwplgPj32QKKVc41ZPiNcBve1HLvAXe3jG0qff60Y1Sqlm5EpPiPbz/zOWd4ELRCTD5bYqpVoJt3pC1N4SlVIO7QlRKeU6t3pCjKu3RO0JUakfB7d6QvwrMEksg4HDxphKl9uqlGolXOkJEaszs+uBPVh/N9+euCYrpc52Yky0yx2bYcYi1cC/W2TmzeNC4JuWbkSC6TK2DU1ZxkuMMY2ex2ixwNLWiUiZMWZgS7cjkXQZ24ZELKNecqqUcp0GFqWU6zSwJM7ylm5AM9BlbBtcX0Y9x6KUcp1+Y1FKuU4DSwKIyH4R+VBEykWkrKXb4wYReU5EvhaRHRFlHURkg4jstofpLdnGMxVjGR8RkS/tbVkuIte3ZBvPhIhcLCJvi8hOEflIRObY5a5vRw0siTPCGJPVhv6qXAlcW6dsLrDRGNMb2Gg/b81WUn8ZAZ60t2WWMeatZm6TmwLAfcaY/sBgYJaI9CcB21EDi4qLMaYE+LZO8U3AKnt8FXBzszbKZTGWsc0wxlSGE7AZY2qAXVhZCFzfjhpYEsMAxSLygYj8uqUbk0CdI+4JOwB0bsnGJFChiGy3fyq16p97YSLSA8gG3iMB21EDS2L8zBiThZVZb5aIDGvpBiWaMU767rbmL0AvIAuoBJa0bHPOnIicC6wD7jHGHImc5tZ21MCSAMaYL+3h18BrWFn42qKqcKZAe/h1C7fHdcaYKmNM0BgTAp6mlW9LEUnGCiovGmNetYtd344aWFwmIueIyHnhceAaYEfD72q1/gpMtscnA2+0YFsSok6K1Txa8ba0k94/C+wyxjwRMcn17agXyLlMRHphfUsBKy3FS8aYhS3YJFeIyMvAcKw7YauAh4HXgdVAd6w71ccZY1rtyc8Yyzgc62eQAfYDd7bWXEMi8jPgHeBDIGQXP4R1nsXV7aiBRSnlOv0ppJRynQYWpZTrNLAopVyngUUp5ToNLEop12lgUUq5TgOLUsp1GliUUq77f1+KAVQOFTOeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f20aedef358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<keras.engine.training.Model at 0x7f20afd2c630>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train(model=model, data=((train_tensors, train_labels), (valid_tensors, valid_labels)), \n",
    "      lr=0.001, lr_decay=0.9, lam_recon=0.392, batch_size=32, shift_fraction=0.1, epochs=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('saved_models/CapsNet.best.from_scratch.hdf5')\n",
    "prediction = eval_model.predict(test_tensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.625410 %\n",
      "Recall: 0.473513 %\n",
      "Fscore: 0.587705 %\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "beta = 0.5\n",
    "\n",
    "pre = K.eval(precision_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))\n",
    "rec = K.eval(recall_threshold(threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))\n",
    "fsc = K.eval(fbeta_score_threshold(beta = beta, threshold = threshold)(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))\n",
    "\n",
    "print (\"Precision: %f %%\\nRecall: %f %%\\nFscore: %f %%\"% (pre, rec, fsc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.62526542"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.eval(binary_accuracy(K.variable(value=test_labels),\n",
    "                                   K.variable(value=prediction[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}