Toxic_Comment_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "from builtins import range\n",
    "# Note: you may need to update your version of future\n",
    "# sudo pip install -U future"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.preprocessing.text import Tokenizer \n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Dense, Input, GlobalMaxPooling1D\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding\n",
    "from keras.models import Model\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the data:\n",
    "# https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge\n",
    "# https://lazyprogrammer.me/course_files/toxic_comment_train.csv\n",
    "# Download the word vectors:\n",
    "# http://nlp.stanford.edu/data/glove.6B.zip \n",
    "# pre-trained glove vectors which were trained using wikipedia data dumps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some configuration\n",
    "MAX_SEQUENCE_LENGTH = 100\n",
    "MAX_VOCAB_SIZE = 20000\n",
    "EMBEDDING_DIM = 100\n",
    "VALIDATION_SPLIT = 0.2\n",
    "BATCH_SIZE = 128\n",
    "EPOCHS = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading word vectors...\n",
      "Found 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "# load in pre-trained word vectors\n",
    "# loading a text file and populating a dictionary\n",
    "print('Loading word vectors...')\n",
    "word2vec = {}\n",
    "# with open(os.path.join('../large_files/glove.6B/glove.6B.%sd.txt' % EMBEDDING_DIM)) as f:\n",
    "with open(os.path.join(r'C:\\Users\\Kirti\\machine_learning_examples\\glove.6B/glove.6B.%sd.txt' % EMBEDDING_DIM), encoding='utf-8') as f:\n",
    "  # is just a space-separated text file in the format:\n",
    "  # word vec[0] vec[1] vec[2] ...\n",
    "  for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    vec = np.asarray(values[1:], dtype='float32')\n",
    "    word2vec[word] = vec\n",
    "print('Found %s word vectors.' % len(word2vec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading in comments...\n"
     ]
    }
   ],
   "source": [
    "# prepare text samples and their labels\n",
    "print('Loading in comments...')\n",
    "\n",
    "# train = pd.read_csv(\"../large_files/toxic-comment/train.csv\")\n",
    "train = pd.read_csv(r\"C:\\Users\\Kirti\\machine_learning_examples\\nlp_class3\\toxic_comment_train.csv\", encoding='utf-8')\n",
    "sentences = train[\"comment_text\"].fillna(\"DUMMY_VALUE\").values\n",
    "possible_labels = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "targets = train[possible_labels].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max sequence length: 5000\n",
      "min sequence length: 6\n",
      "median sequence length: 205\n"
     ]
    }
   ],
   "source": [
    "print(\"max sequence length:\", max(len(s) for s in sentences))\n",
    "print(\"min sequence length:\", min(len(s) for s in sentences))\n",
    "s = sorted(len(s) for s in sentences)\n",
    "print(\"median sequence length:\", s[len(s) // 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert the sentences (strings) into integers\n",
    "tokenizer = Tokenizer(num_words=MAX_VOCAB_SIZE)\n",
    "tokenizer.fit_on_texts(sentences)\n",
    "sequences = tokenizer.texts_to_sequences(sentences)\n",
    "# print(\"sequences:\", sequences); exit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max word index: 19999\n"
     ]
    }
   ],
   "source": [
    "print(\"max word index:\", max(max(seq) for seq in sequences if len(seq) > 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 210337 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "# get word -> integer mapping\n",
    "word2idx = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word2idx))\n",
    "\n",
    "# exit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of data tensor: (159571, 100)\n"
     ]
    }
   ],
   "source": [
    "# pad sequences so that we get a N x T matrix\n",
    "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "print('Shape of data tensor:', data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filling pre-trained embeddings...\n"
     ]
    }
   ],
   "source": [
    "# prepare embedding matrix\n",
    "print('Filling pre-trained embeddings...')\n",
    "num_words = min(MAX_VOCAB_SIZE, len(word2idx) + 1)\n",
    "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
    "for word, i in word2idx.items():\n",
    "  if i < MAX_VOCAB_SIZE:\n",
    "    embedding_vector = word2vec.get(word)\n",
    "    if embedding_vector is not None:\n",
    "      # words not found in embedding index will be all zeros.\n",
    "      embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load pre-trained word embeddings into an Embedding layer\n",
    "# note that we set trainable = False so as to keep the embeddings fixed\n",
    "embedding_layer = Embedding(\n",
    "  num_words,\n",
    "  EMBEDDING_DIM,\n",
    "  weights=[embedding_matrix],\n",
    "  input_length=MAX_SEQUENCE_LENGTH,\n",
    "  trainable=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building model...\n"
     ]
    }
   ],
   "source": [
    "print('Building model...')\n",
    "\n",
    "# train a 1D convnet with global maxpooling\n",
    "input_ = Input(shape=(MAX_SEQUENCE_LENGTH,))\n",
    "x = embedding_layer(input_)\n",
    "x = Conv1D(128, 3, activation='relu')(x)\n",
    "x = MaxPooling1D(3)(x)\n",
    "x = Conv1D(128, 3, activation='relu')(x)\n",
    "x = MaxPooling1D(3)(x)\n",
    "x = Conv1D(128, 3, activation='relu')(x)\n",
    "x = GlobalMaxPooling1D()(x)\n",
    "x = Dense(128, activation='relu')(x)\n",
    "output = Dense(len(possible_labels), activation='sigmoid')(x)\n",
    "\n",
    "model = Model(input_, output)\n",
    "model.compile(\n",
    "  loss='binary_crossentropy',\n",
    "  optimizer='rmsprop',\n",
    "  metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model...\n",
      "Epoch 1/10\n",
      "998/998 [==============================] - 72s 72ms/step - loss: 0.0850 - accuracy: 0.9834 - val_loss: 0.0719 - val_accuracy: 0.9940\n",
      "Epoch 2/10\n",
      "998/998 [==============================] - 70s 70ms/step - loss: 0.0675 - accuracy: 0.9926 - val_loss: 0.0775 - val_accuracy: 0.9937\n",
      "Epoch 3/10\n",
      "998/998 [==============================] - 73s 73ms/step - loss: 0.0632 - accuracy: 0.9928 - val_loss: 0.0767 - val_accuracy: 0.9886\n",
      "Epoch 4/10\n",
      "998/998 [==============================] - 75s 75ms/step - loss: 0.0598 - accuracy: 0.9932 - val_loss: 0.0773 - val_accuracy: 0.9935\n",
      "Epoch 5/10\n",
      "998/998 [==============================] - 65s 65ms/step - loss: 0.0575 - accuracy: 0.9934 - val_loss: 0.1079 - val_accuracy: 0.9940\n",
      "Epoch 6/10\n",
      "998/998 [==============================] - 63s 63ms/step - loss: 0.0557 - accuracy: 0.9931 - val_loss: 0.0726 - val_accuracy: 0.9932\n",
      "Epoch 7/10\n",
      "998/998 [==============================] - 65s 65ms/step - loss: 0.0539 - accuracy: 0.9934 - val_loss: 0.0760 - val_accuracy: 0.9941\n",
      "Epoch 8/10\n",
      "998/998 [==============================] - 65s 65ms/step - loss: 0.0528 - accuracy: 0.9933 - val_loss: 0.0817 - val_accuracy: 0.9939\n",
      "Epoch 9/10\n",
      "998/998 [==============================] - 62s 62ms/step - loss: 0.0514 - accuracy: 0.9933 - val_loss: 0.0764 - val_accuracy: 0.9938\n",
      "Epoch 10/10\n",
      "998/998 [==============================] - 51s 51ms/step - loss: 0.0496 - accuracy: 0.9934 - val_loss: 0.1167 - val_accuracy: 0.9940\n"
     ]
    }
   ],
   "source": [
    "print('Training model...')\n",
    "r = model.fit(\n",
    "  data,\n",
    "  targets,\n",
    "  batch_size=BATCH_SIZE,\n",
    "  epochs=EPOCHS,\n",
    "  validation_split=VALIDATION_SPLIT\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9bn48c+TPWRjSwhJ2AkkgSACImrFBRWhGlrxWqzVn7bVa61WvVertretbe21rb2tesvVS1vb2mLVK7RFRRQQxV0WwxogIbIkAZKwhCSQ/fv74zsJkzCBCczkzPK8X6+8JjnnzJwnQ3jmnOc85/sVYwxKKaVCV4TTASillPIvTfRKKRXiNNErpVSI00SvlFIhThO9UkqFOE30SikV4rxK9CJytYhsF5ESEXnYw/ocEflIRBpF5AG35UNEZJWIFInIFhG515fBK6WUOj05XR+9iEQCO4ArgTJgDXCjMWar2zZpwDDgS8BhY8yvXMsHA4ONMetFJAlYB3zJ/blKKaX8y5sj+qlAiTGm1BjTBLwIzHHfwBhTaYxZAzR3Wb7PGLPe9X0tUARk+iRypZRSXonyYptMYK/bz2XA+T3dkYgMB84FPulm/R3AHQAJCQmTc3JyeroLpZQKW+vWras2xqR6WudNohcPy3o0boKIJAKLgPuMMUc9bWOMWQAsAJgyZYpZu3ZtT3ahlFJhTUR2d7fOm9JNGTDE7ecsoKIHO4/GJvmFxpjF3j5PKaWUb3iT6NcA2SIyQkRigHnAEm9eXEQE+ANQZIz59ZmHqZRS6kydtnRjjGkRkbuBN4FI4DljzBYRudO1/lkRSQfWAslAm4jcB+QBE4CbgU0iUuh6ye8ZY5b64XdRSinlgTc1elyJeWmXZc+6fb8fW9Lp6n081/h7rLm5mbKyMhoaGnzxciErLi6OrKwsoqOjnQ5FKRUgvEr0gaCsrIykpCSGDx+OrQiprowxHDx4kLKyMkaMGOF0OEqpABE0QyA0NDQwYMAATfKnICIMGDBAz3qUUp0ETaIHNMl7Qd8jpVRXQZXolVIqZG1fBh/Nh9YWn7+0JvoeSExMdDoEpVSoWvdH+HQBRET6/KU10SullNOaG+Dz1TD6SvBD+VUT/RkwxvDggw8yfvx48vPzeemllwDYt28f06dPZ+LEiYwfP5733nuP1tZWbr311o5tf/Ob3zgcvVIq4Oz5EJqPQfaVfnn5oGmvdPfjV7ewtcLjkDlnLC8jmR9dO86rbRcvXkxhYSEbNmygurqa8847j+nTp/PCCy8wc+ZMvv/979Pa2sqxY8coLCykvLyczZs3A3DkyBGfxq2UCgElKyEyFoZf7JeX1yP6M/D+++9z4403EhkZyaBBg7jkkktYs2YN5513Hn/84x959NFH2bRpE0lJSYwcOZLS0lLuueceli1bRnJystPhK6UCTfFyGH4RxPTxy8sH5RG9t0fe/tLdZC3Tp09n9erVvP7669x88808+OCD3HLLLWzYsIE333yT+fPn8/LLL/Pcc8/1csRKqYB1ZA9Ub4fJ/89vu9Aj+jMwffp0XnrpJVpbW6mqqmL16tVMnTqV3bt3k5aWxu233843vvEN1q9fT3V1NW1tbcydO5ef/vSnrF+/3unwlVKBpHi5fRztn/o8BOkRvdO+/OUv89FHH3HOOecgIvzyl78kPT2dP//5zzzxxBNER0eTmJjI888/T3l5ObfddhttbW0APP744w5Hr5QKKCUroO9QGJjtt12cds5YJ3iaeKSoqIjc3FyHIgou+l4pFSRamuAXw+GceXDN2Y3kLiLrjDFTPK3T0o1SSjllz0fQXO+3tsp2muiVUsopJcshMsZvbZXtNNErpZRTilfA0Asg1r/Dq2iiV0opJ9SUQVWR38s2oIleKaWcUbLCPvqxrbKdJnqllHJC8XJIGQKpY/2+K030SinV21qaoPRdGD3DL6NVdqWJ3k9ONXb9rl27GD9+fC9Go5QKKHs/gabaXinbgJeJXkSuFpHtIlIiIg97WJ8jIh+JSKOIPNBl3XMiUikim30VtFJKBbWS5RARDSMv6ZXdnXYIBBGJBOYDVwJlwBoRWWKM2eq22SHgO8CXPLzEn4DfAs+fdbTt3ngY9m/y2csBkJ4Ps37e7eqHHnqIYcOGcddddwHw6KOPIiKsXr2aw4cP09zczGOPPcacOXN6tNuGhga+9a1vsXbtWqKiovj1r3/NZZddxpYtW7jttttoamqira2NRYsWkZGRwQ033EBZWRmtra384Ac/4Ctf+cpZ/dpKKQeUrISh0yA2qVd2581YN1OBEmNMKYCIvAjMAToSvTGmEqgUkS92fbIxZrWIDPdJtA6aN28e9913X0eif/nll1m2bBn3338/ycnJVFdXM23aNAoKCno0Qff8+fMB2LRpE9u2beOqq65ix44dPPvss9x7773cdNNNNDU10draytKlS8nIyOD1118HoKamxve/qFLKv45WwIHNcOVPem2X3iT6TGCv289lwPm+DkRE7gDuABg6dOipNz7Fkbe/nHvuuVRWVlJRUUFVVRX9+vVj8ODB3H///axevZqIiAjKy8s5cOAA6enpXr/u+++/zz333ANATk4Ow4YNY8eOHVxwwQX87Gc/o6ysjOuuu47s7Gzy8/N54IEHeOihh7jmmmu4+GL/3k2nlPKDjrbKK3ptl97U6D0dnvp8JDRjzAJjzBRjzJTU1FRfv7xPXH/99bzyyiu89NJLzJs3j4ULF1JVVcW6desoLCxk0KBBNDQ09Og1uxtU7qtf/SpLliwhPj6emTNn8vbbbzNmzBjWrVtHfn4+jzzyCD/5Se8dESilfKR4OSRlQFper+3SmyP6MmCI289ZQIV/wgls8+bN4/bbb6e6upp3332Xl19+mbS0NKKjo1m1ahW7d+/u8WtOnz6dhQsXcvnll7Njxw727NnD2LFjKS0tZeTIkXznO9+htLSUjRs3kpOTQ//+/fna175GYmIif/rTn3z/Syql/Ke1GUrfgXFf6pW2ynbeJPo1QLaIjADKgXnAV/0aVYAaN24ctbW1ZGZmMnjwYG666SauvfZapkyZwsSJE8nJyenxa951113ceeed5OfnExUVxZ/+9CdiY2N56aWX+Otf/0p0dDTp6en88Ic/ZM2aNTz44INEREQQHR3NM88844ffUinlN2VroPFor7VVtvNqPHoRmQ08CUQCzxljfiYidwIYY54VkXRgLZAMtAF1QJ4x5qiI/A24FBgIHAB+ZIz5w6n2p+PRnx19r5QKUCt+DB8+Dd/9HOJ8O3/0qcaj92qGKWPMUmBpl2XPun2/H1vS8fTcG70PVSk/2fAS1FfChfc4HYkKZyXLYcg0nyf509GpBP1o06ZN3HzzzZ2WxcbG8sknnzgUUZhqbYa3/gMaamDyrb3Wu6xUJ7X77f0/M37U67sOqkRvjOlRj7rT8vPzKSws7NV9BuLUkI4rWWmP5gGK34Lxc52NR4Wn9rbKXhiWuKugGesmLi6OgwcPaiI7BWMMBw8eJC4uzulQAkvhQugzEBLSYOsSp6NR4apkBSQNhkG9P85V0BzRZ2VlUVZWRlVVldOhBLS4uDiysjxeLglPxw7B9jdg6u3QfBw2vmwfo+OdjkyFk9YW2Pk25F7bq22V7YIm0UdHRzNixAinw1DBZtMr0NYME78K9VWw7o+2lJN7jdORqXBSvtZeI+rltsp2QVO6UeqMFC60A9al59sJmOP6QpGWb1QvK14OEgkjL3Vk95roVeg6sBX2FcLEm+zPkdGQ80XYvsxO/KBUbylZDkOmQnxfR3aviV6Frg0vQEQU5P/LiWW5BdBYA5+/61xcKrzUVcK+Db06iFlXmuhVaGptsTdJjbkaEgaeWD7qMohJgq3/dC42FV5KVtpHB9oq22miV6Fpp6t3fmKXYZmiYmHMTNj2uv0wUMrfSpZD4iBIn+BYCJroVWhq753PvurkdXkFcPwQ7P6g9+NS4aWt1bZVjuqdScC7o4lehZ723vkJN9gLsF2NvgKi4rX7Rvlf+To4fhiynavPgyZ6FYo2L4LWppPLNu1iEux/vKLXoK2td2NT4aVkBUgEjLzM0TA00avQ4947353cOVC3H8o+7b24VPgpXg5Z50Gf/o6GoYlehZYDW6HisxO9890ZMxMiY3TsG+U/9dX2b9Ghu2HdaaJXocVT77wnccn2dLroVdCB8pQ/lKwEDIye4XQkmuhVCOmud747eQVQs8cedSnlayXLISEVBk90OhJN9CqEdNc7352xs+34I9p9o3ytrdUe0Y+aARHOp1nnI1DKV07VO+9Jn/4w4mJbp9fyjfKlikJ7r4aDd8O600SvQkN773z+v3june9ObgEc2gmVW/0Xmwo/JcsBgVGXOx0J4GWiF5GrRWS7iJSIyMMe1ueIyEci0igiD/TkuUr5xOl657uTcw0g2n2jfKt4OWROdrytst1pE72IRALzgVlAHnCjiOR12ewQ8B3gV2fwXKXOXuFCGJQPg3s4nkjSIBh6gdbple/UH7R3xAZI2Qa8O6KfCpQYY0qNMU3Ai8Ac9w2MMZXGmDVAc0+fq9RZqyxy9c738Gi+XV6BLd1Ul/g2LhWeSldh2yqDK9FnAnvdfi5zLfOG188VkTtEZK2IrNV5YVWPFHrZO9+d3GvtY5EOXax8oHg59BkAGec6HUkHbxK9pyHXvG1R8Pq5xpgFxpgpxpgpqampXr68CnutLbDxJcieCYln+HeTkgWZU7ROr85eW5sd3yZA2irbeRNJGTDE7ecsoMLL1z+b5yp1ejvfhroDZ162aZdXYKcdPLzbN3Gp8LSvEI5VOzqblCfeJPo1QLaIjBCRGGAe4O2hz9k8V6nTK1xoT5O97Z3vTm6BfSx69exjUuGrZAUgATHsgbvTJnpjTAtwN/AmUAS8bIzZIiJ3isidACKSLiJlwL8B/yEiZSKS3N1z/fXLqDBz7BBsXwr5N0BUzNm9Vv8RdrRL7b5RZ6Nkha3NezMERy+K8mYjY8xSYGmXZc+6fb8fW5bx6rlK+cSZ9s53J3cOrHoMju6D5MG+eU0VPo4dgrI1MP1BpyM5SeBcLVCqpwpfOLPe+e7kuco3217zzeup8FK6CkxbQLVVttNEr4JTZRFUrPfd0TxA6lgYOBa2apulOgPFKyC+H2ROcjqSk2iiV8HpbHvnu5NXYCcNr6/27euq0NbRVnk5REQ6Hc1JNNGr4OOL3vnu5BbY0+9tr/v2dVVoO7DJDpEdgGUb0ESvgpGveuc9Sc+HfsO1+0b1TPFy+xhgbZXtNNGr4OOr3nlPROxRfem7cPyI719fhaaSFXYmqcQ0pyPxSBO9Ci6+7J3vTt4caGuGHcv88/oqtBw/Ans/Dbi7Yd1polfBpaN3/kb/7SNjEiRn6tg3yjulq8C0BtSwxF1polfBpfAFGDQe0n3UO+9JRIQd0XLnSmis899+VGgoWQFxKXZgvACliV4Fj8ptJ3rnxdPAqD6UWwAtDVD8ln/3o4KbMa5JwC+HSK8GGnCEJnoVPDa0987f4P99DZ0GCanafaNO7cBmqN0XsG2V7TTRq+DQ2gIbXrKdNr7unfckItLOJ7vjLWg+7v/9qeAU4G2V7TTRq+BQugrq9vund747eQXQXG/79pXypGSFvfciKd3pSE5JE70KDoULIb6/vRu2twy/GOL6aveN8qyhBvZ+EvBlG9BEr4LB8cN2SIIJfuyd9yQyGnK+CNvfgJam3tuvCg6l70JbS0C3VbbTRK8Cn6/Hne+J3AJorIHPV/f+vlVgK1kOsSmQNdXpSE5LE70KfL3RO9+dUZdBTBIU6dDFyo0xdljikZcEdFtlO030KrBVboPydb3TO+9JVCyMmWlLR60tvb9/FZgqt0JtRVCUbUATvQp0vdk73528Ajh2EPZ86FwMKrCUrLCPATy+jTtN9Cpw9XbvfHdGXwFR8dp9o04oXm7LickZTkfiFU30KnA50TvvSUwCZF8BRa/amYRUeGushT0fB83RPHiZ6EXkahHZLiIlIvKwh/UiIk+71m8UkUlu6+4Vkc0iskVE7vNl8CrEOdE7353cOfZDp2yN05Eop5W+a4exDqVELyKRwHxgFpAH3CgieV02mwVku77uAJ5xPXc8cDswFTgHuEZEsn0WvQpdTvXOd2fMTIiM0bFvlG2rjEmy4yEFCW+O6KcCJcaYUmNME/AiMKfLNnOA5431MdBXRAYDucDHxphjxpgW4F3gyz6MX4UqJ3vnPYlLhpGX2Tq9MU5Ho5zSPlrlyEvsDXVBwptEnwnsdfu5zLXMm202A9NFZICI9AFmA0M87URE7hCRtSKytqqqytv4VagqfAHSxjnTO9+dvAKo2QMVnzkdiXJK1Xao2Rs0bZXtvEn0npqXux7SeNzGGFME/AJYDiwDNgAem5GNMQuMMVOMMVNSUx3ssFDOq9rubO98d8bOBonU8k04K2kfrTL0En0ZnY/Cs4AKb7cxxvzBGDPJGDMdOAQUn3m4KiwUvmAT6gQHe+c96dMfRlys5ZtwVrwc0vIgpWtRI7B5k+jXANkiMkJEYoB5QNdDmiXALa7um2lAjTFmH4CIpLkehwLXAX/zWfQq9LS1wsb23vk0p6M5WW4BHNpp74xU4aWxDvZ8FPBjz3ty2kTvuoh6N/AmUAS8bIzZIiJ3isidrs2WAqVACfA74C63l1gkIluBV4FvG2MO+/IXUCFm5yo7Y0+gXITtKucaQPTmqXC06z3bIBBkZRsAr0bjMcYsxSZz92XPun1vgG9389yLzyZAFWbae+fHXO10JJ4lDYKhF9g6/WWPOB2N6k3FyyEm0f77Bxm9M1YFjvbe+fx/CYze+e7kFdjSTXWJ05Go3mKMvRA74pLA/tvshiZ6FTg2L4bWxsAt27TLvdY+6tDF4aO6GI7ssUNhBCFN9CpwtPfODz7H6UhOLSULMidrnT6cdLRVaqJX6sxVbYfytYHXO9+d3ALYVwiHdzsdieoNJStg4FjoO9TpSM6IJnoVGAK1d747eQX2sehVZ+NQ/td0DHZ9EHR3w7rTRK+cF+i98570HwmD8vUu2XCw6z177ShIyzagiV4FgkDvne9OXgHs/QSO7nM6EuVPxcshug8Mu9DpSM5YSCX6yqMN1DfqvJ5BJ9B757uT6yrfbHvN2TiU/3S0VU638wcHqZBJ9DXHmpn55GqeeHO706GongiW3nlP0nJg4BjYqm2WIetQKRzeFdRlGwihRJ/SJ5qCczL480e7WLvrkNPhKG8FS+98d3ILYPcHUF/tdCTKH4pdbZVBfCEWQijRA3z36hwyUuL57isbaWhudToc5Y1g6Z3vTl4BmDZ7VqJCT8lyGJAN/YY7HclZCalEnxAbxS/mTqC0up4nV+hoyAGvakdw9c57kj4B+g7T7ptQ1Hwcdr0f9EfzEGKJHuAL2QP5ypQhLFi9kw17jzgdjjqVDUHWO++JiD2qL30XjuvfW0jZ9T60NATlsMRdhVyiB/jeF3NJTYrloUUbaWppczoc5UlbK2x40R4tBUvvfHdy50BbM+xY5nQkypdKVkBUPAz7gtORnLWQTPQp8dH855fz2ba/lv95R0cYDEilQdo770nmZEjK0LFvQk3xcjujWHSc05GctZBM9AAzcgcxZ2IGv327hKJ9R50OR3VV+ALE9wu+3nlPIiLsiJY7V9pZiFTwO1RqZxILwklGPAnZRA/wo2vHkRIfzXdf2UhLq5ZwAsbxI1D0mqt3PnhvQukkr8DWc4vfcjoS5QvFK+xjkA5L3FVIJ/r+CTH8ZM54NpXX8Pv3P3c6HNVuS5D3znsy9AJISNXum55obYGKz6ClyelITlay3I5n1H+k05H4REgneoDZ+enMHDeIXy/fwc4qPa0OCIUvQFoeDJ7odCS+ExEJOV+EHW/ZtjzVvdZmWP8X+O1kWHApPD0RPvwtNNY6HZnV3ACfvxcyZRsIg0QvIvx0znjioyN56JWNtLUZp0MKb1U7oGxNcPfOdye3AJrrYefbTkcSmFoaYe1z8PQkWHI3xPWF2b+yR81vfR9+Mw5W/hTqKp2Nc/cH0HI8JPrn23mV6EXkahHZLiIlIvKwh/UiIk+71m8UkUlu6+4XkS0isllE/iYivX4JOy05jh9ek8fa3Yd5/qNdvb175a69dz4/iHvnuzNiuk1e2n3TWXMDfLIAnj4XXrvfttN+9f/gjndg6u1w62vwzbftfKzv/Rc8mW+3O1TqTLwlKyAqDoYHf1tlu9MmehGJBOYDs4A84EYRyeuy2Swg2/V1B/CM67mZwHeAKcaY8UAkMM9n0ffAdZMyuWRMKr9Ytp29h445EYJy751PGuR0NL4XGQ1jZ8P2NwKz7tzbmo7BR/PhqQnwxoN2dqab/w7fXAFjrup8Rpc1Gb7yF7h7LUz4Cnz2V/jvyfB/t9o6fm8qXm6TfHR87+7Xj7w5op8KlBhjSo0xTcCLwJwu28wBnjfWx0BfERnsWhcFxItIFNAHqPBR7D0iIvzndflERggPL96IMVrC6XWh1DvfnbwCaKyBz1c7HYlzGuvg/Sdtgn/ze3aEz//3Gtz2Boy6/NQlu4GjoeBpuG8TXHQvlKy0dfw/F9iSmL//3x7eBQeLg360yq68SfSZwF63n8tcy067jTGmHPgVsAfYB9QYYzz2n4nIHSKyVkTWVlVVeRt/j2T2jeeR2Tl8UHKQl9bsPf0TlG+FUu98d0ZeBjGJUBSGQxc3HIXVv7KllxU/gvR8uG2ZLc2MuLhn12SS0uGKR+H+LXDlT+ycwn/5Miy4BDYvsh07/lDiaqsMoQux4F2i9/Sv0/Vj1eM2ItIPe7Q/AsgAEkTka552YoxZYIyZYoyZkpqa6kVYZ+bG84YybWR/fvZ6EftqtDui14Ri77wn0XEwZqYdzdJfySjQHD8M7/wcnhwPb/8Uss6Db660ZZphF5zda8cl2yP7+zZCwX/bctArX7cdO5/+zvcdTsUr7EiVA0b59nUd5k2iLwOGuP2cxcnll+62uQL43BhTZYxpBhYDjs7HFREh/GLuBJrb2vje4k1awuktodg7353cAjh2EPZ86HQk/nXskO2SeXICvPM4DL/YXmC96WXImuLbfUXFwqRb4NufwlcW2nsWlj4AvxkP7z5hYzlbLY225Db6ypDrCPMm0a8BskVkhIjEYC+mdm0rWALc4uq+mYYt0ezDlmymiUgfERFgBlDkw/jPyLABCTw4M4dV26v4R2G50+GEh1Dsne9O9pV2MKxQ7b6pq4LlP7RJ9r3/snX3Oz+AeQsh41z/7jsiAnKvgW8sh1uXQuYkWPWYjWXZ96Cm7Mxfe/eHtj02hNoq20WdbgNjTIuI3A28ie2aec4Ys0VE7nStfxZYCswGSoBjwG2udZ+IyCvAeqAF+AxY4I9fpKduvXA4r2+s4MevbuULo1NJTQrhcoLTqott7/xVj4XckZJHMQl2aNuiV2HWL21yCgW1++GDp20vfGsjjLsOpj8Aabm9H4sIDL/Ifh3YAh88BZ88C5/+ry0PXnRvz+MqWQGRMSHVVtlOArF0MWXKFLN27Vq/76ekspbZT73PFXlp/M9Nk/2+v7C14sf2P+K/FYVmW6UnG1+GxbfD19+Coec7Hc3ZqSm3/37r/gRtLXb+gIv/HQZmOx1ZZ0f2wEf/A+v/DM3H7EX/i+7z/jrB/PMhaTDc8g//xuknIrLOGOOxZhYihxpnZnRaEvdekc3STft5Y9M+p8MJTe2986OvCJ8kD/aCbER0cI99c2SPvXHp6Ymw9g82wd+zFr78bOAlebB9+rN+bjt1Lv2ePYv849Xwh6vsxfG2UwxseGQvVG0LybINeFG6CXV3TB/J0k37+ME/tzBt5AD6JcSc+Ys1N8Bnf4F1fwYMxKXYOyXjUuxXvNv37cvdl8UkBldpwxh75NRYa3unG49CU53rZ9fXwZ1QWwFXP+50tL0rLsXWrrcuCb6S1aFSeO/XsOFvgMCkm+2Rcb9hTkfmnT794dKH4MJ7oHAhfPg0vPhVGDgWLvqOvSs7qsv/8xLXJOAh1lbZLuwTfXRkBL+8fgJzfvsBP31tK7/+yhlcLGyqt6e1HzwNdfvtRBSJ6dBQA0d228eGGpsIT0UiT/2h0LG8r+fl3rQtGmOH03VPxl2Tc6dlrgR+0rJaaKq1E2OfTr8RMHaWV29lSMkrgOI3YV+h/y9S+kJ1Cbz3K1t2ioiCKV+3te6ULKcjOzMxfewQC5Nvg63/sDdx/fPb8PZjMO0umHyrbd8E21bZd2hgnqn4QNgneoBxGSncdekonn67hGvPyeCyHC+ntms4Cmt+b2/zPlZt28vm/s4+ejqCa22xSbOhBhqOnPgAOO72ffvy9mVH951Y1tJw6nii4jp/AMQknHzE3VgLptWLX04gNunEV0yifUxKh9jkzstiEz0sc/uKiguuI1pfGTvbfnhvXRLYib5yG6x+wrbARsbC+XfaI9+kdKcj843IKMi/HsbPtZPDvP8kLP+BvbnrvG/Yr8/ftaWpEP07DeuLse4aW1q55un3qWts4c37p5McF939xscPwyf/Cx8/Y5Pw6Ctg+oMwdJp/g2xucPtAcP+wOOL2YeH2odFYZ49qvErMXZbFJITsH32ven6Obfm7e23gvZ/7N9sEv/WfEN0Hpn4TLrgHEv13w2LAKF9nLzBvXWL/XUwbzPsb5Mx2OrIzdqqLsXpE7xIbFckvr5/A3Gc+5PGl23j8uvyTN6qvtkfvn/7Oli3GftG2l2VOOnlbf4iOs1/hdFEz2OUWwOv/BpVFMKjrWIAOqD9oZ8Ha+g87mXlMku2gmXYXJAxwOrrekzkZbnjeXkP68Gn77zPyEqej8htN9G7OHdqPb148kgWrS7l2wmAuHD3QrqjdDx/+t+0fbj4O474EFz8A6eOdDVgFvpxr4PV/t903TiR6Y+x9DDvesKNq7v3EHr0mpsMlD8O0O+34Q+FqwCi49imno/A7Ld10cbyplVlPrabVGN76+ijiP/2t7aJpa7E3Ylz8b5A61pHYVJB6bpYtpd3VS0MitLbA3o9tYt/+hp3kGuwgY2Nn2/7ywRND50YuBWjppkfiYyJ58qq+bP2/HxMz/z07XNs5N9oEHyLzRyLPaHwAABUXSURBVKpellcAyx62ZQJ/DZbVUGPv7Ny+zJZmGo7YuzxHTIdp37LJve+Q07+OCkma6N1VF8N7v2bixpcYHx3BCy2Xcc5XfsiE8ROcjkwFs9xrbaLf+k97wOArh3fZxL59qZ3+rq0F+gywR+1jZ8Goy+zFdRX2NNEDHNhq+4c3L7atgOffSeOUb/Hs74uJffMQS8e2Ehcd6XSUKlilZNmLf0VLzi7Rt7XZbpH2envlVrt84Fi44G6b3LPOsxOVK+UmvBN9RaFtL9v2mm0pvOhe+x8mMZUE4PHr4rjluU95amUxD12d43S0KpjlFtjJOI7ssTfmeKupHkrfsUftO96C+krbmz/sQpj5n7YkE2JjpyvfC89Ev3cNrP6lrWXGpsAlD9mbRPr077TZ9DGp3DAliwWrS5k1Pp0JWX0dClgFvTxXoi96FS749qm3PVphWx+3L7NJvrXR/p1mXwFjZtnHcO6UUT0WXol+1/v2CL70HYjvD5f/wN4iHZfS7VO+/8U83tlexXdf2ciSu79ATJR2Kqgz0H8kDMq3N+h0TfTGwP6NJ+rt+wrt8r7D7DAEY2fZI/jIU9zEp9QphH6iN8ZOSv3uE3bGn4Q0O8jU5Nvs3aCnkRIfzc++nM/tz6/lmXd2cu8VoTkWhuoFeQWw6j/tfRnx/eDz91z19mVwtAwQW2Of8SOb3FNzAu9uWhWUQjfRGwM73rRH8OVrISnDTgIx6RaIju/RS12ZN4iCczL47apiZo4fRE56sp+CViEttwBW/QwWXg+HPreDxEX3saNcXvYIZM8Mj+EHVK8LvUTf1gbbXrUJfv8me+HrmiftXKVnMSn1owXj+KCkmu++spHF37qQqEgt4ageSsuBrKlQs9cOoDVmlu1zj45zOjIV4kIn0be12vbI935lJxAYMBq+9Iy9m9UHtc3+CTE8WjCOe/72GX94/3P+9RLtdFBn4Btv2UctyaheFDqHpc3H4Y0HAYG5f7CzxU/8qk8vYF0zYTBX5Q3iv5bvYGdVnc9eV4UREU3yqteFTqKPTYRvrIBvfWjHnvbDTSMiwmNfGk9cVAQPL9pIW1vgjROklFJdhU6iBxg42u8DNaUlx/GDa/JYs+swf/l4t1/3pZRSvuBVVhSRq0Vku4iUiMjDHtaLiDztWr9RRCa5lo8VkUK3r6Micp+vf4nedv3kLKaPSeUXy7ax99Axp8NRSqlTOm2iF5FIYD4wC8gDbhSRrgNrzwKyXV93AM8AGGO2G2MmGmMmApOBY8DffRe+M0SEx6/LR4BHFm8iEId6Vkqpdt4c0U8FSowxpcaYJuBFYE6XbeYAzxvrY6CviAzuss0MYKcxJiTqHZl943l4di7vl1Tz8tq9ToejlFLd8ibRZwLumazMtayn28wD/tbdTkTkDhFZKyJrq6qqvAjLeTdNHcr5I/rz2GtF7K85zcTdSinlEG8SvadesK61ilNuIyIxQAHwf93txBizwBgzxRgzJTU1OO4OjIgQfjF3As1tbXz/71rCUUoFJm8SfRngPjVNFlDRw21mAeuNMQfOJMhANnxgAg9cNZaV2ypZsqHr26KUUs7zJtGvAbJFZITryHwesKTLNkuAW1zdN9OAGmPMPrf1N3KKsk2wu+2iEUwc0pcfLdlCVW2j0+EopVQnp030xpgW4G7gTaAIeNkYs0VE7hSRO12bLQVKgRLgd8Bd7c8XkT7AlcBiH8ceMCIjhCeun8CxxlYeXbLF6XCUUqoTr8a6McYsxSZz92XPun1vAI+zKRhjjgEDziLGoJA9KInvzBjNr97awbWb93H1+K5NR0op5YzQujPWYf96ySjyBifzH//YQvGBWqfDUUopQBO9T0VHRvDEv0ygrrGZK3+zmjnzP+AvH+3iyLEmp0NTSoUxCcSWwClTppi1a9c6HcYZq6xtYElhBa+sK2Pb/lpiIiOYkZvG3ElZXDI2lWgdy14p5WMiss4YM8XjOk30/mOMYUvFURatL2NJYQUH65sYmBhDwTmZzJ2cybiM7ueqVUqpntBEHwCaW9t4Z3sVi9aVsXLbAZpbDTnpSVw/OYs5EzNJTTrz2a+UUkoTfYA5XN/EqxsrWLSujA1lNURGCJeMSWXupCxm5KYRF+37sfSVUqFNE30AK6ms5ZV15fz9szIOHG0kJT6aayYMZu7kLM4d0hfR2YiUUl7QRB8EWtsMH5RUs2h9Gcs276expY2RqQnMnZTFdZMyGZwS73SISqkApok+yNQ2NLN00z4WrSvn012HEIGLRg1k7uRMZo5Lp09M6MzprpTyDU30QWz3wXoWry9n0foyyg4fJyEmktn5trQzdXh/IiK0tKOU0kQfEtraDJ/uOsSidWUs3bSP+qZWsvrFc92kLOZOymTYgASnQ1RKOUgTfYg51tTCm1v2s2hdOR/srMYYOG94P+ZOymL2hMEkx0U7HaJSqpdpog9hFUeO8/fPbGmntKqe2KgIZo5LZ+7kLL4weiCRWtpRKixoog8DxhgK9x5h0foyXt2wj5rjzaQlxXJF3iCuyE3jwlEDtT9fqRCmiT7MNLa0srKokn8WlvNecTXHmlqJi47gC6NTuSI3jctz0khLjnM6TKWUD50q0WufXgiKjbKdObPzB9PQ3Monnx9iZdEBVhZVsqLIzuY4ISuFGTmDmJGbxriMZL0xS6kQpkf0YcQYw7b9tTbpb6ukcO8RjIHBKXFcnpPGFbmDuGDUAC3xKBWEtHSjPKqqbWTV9kpWFh3oKPHER0dy0eiBtsSTm0ZakpZ4lAoGmujVaTU0t/Jx6UHe3lbJyqJKyo8cB+CcrBRm5NoST95gLfEoFag00asecS/xrCiqZEOZlniUCnRnnehF5GrgKSAS+L0x5udd1otr/WzgGHCrMWa9a11f4PfAeMAAXzfGfHSq/WmiDyzdlXi+kG1LPJflaIlHKaedVdeNiEQC84ErgTJgjYgsMcZsddtsFpDt+jofeMb1CPYDYJkx5noRiQH6nPFvohyRmhTLDVOGcMOUIR0lnpVFNvEv32q7eM4Z0pcZOWla4lEqAJ32iF5ELgAeNcbMdP38CIAx5nG3bf4XeMcY8zfXz9uBS4F6YAMw0vSgRqRH9MHBGEPRvhNdPO0lnoyUOC7PTWNG7iAuGKklHqV6w9n20WcCe91+LuPE0fqptskEWoAq4I8icg6wDrjXGFPvIcg7gDsAhg4d6kVYymkiQl5GMnkZydwzI9uWeLbZXv1F68r568d76BMTybSRA5iQlcL4jBTGZ6YwKDlWj/iV6kXeJHpP/yO7Hp13t00UMAm4xxjziYg8BTwM/OCkjY1ZACwAe0TvRVwqwKQmxXLDeUO44bzOJZ4Pd1azansl7ed0AxNjGZ+Z7Er8yYzLSCGrX7wmf6X8xJtEXwYMcfs5C6jwchsDlBljPnEtfwWb6FWIi4uO5NKxaVw6Ng2A+sYWivYdZXN5DZsr7ON7xdW0ttnsnxIf3ZH8x2WmMD4jmeEDEnS8faV8wJtEvwbIFpERQDkwD/hql22WAHeLyIvYsk6NMWYfgIjsFZGxxpjtwAxgKyrsJMRGMWV4f6YM79+xrKG5le37a9lcUcPm8qNsqajhjx/soqm1DYDE2CjyMk4c+Y/PTGHkwASiIiOc+jWUCkqnTfTGmBYRuRt4E9te+ZwxZouI3Ola/yywFNtaWYJtr7zN7SXuARa6Om5Ku6xTYSwuOpJzhvTlnCF9O5Y1tbRRXFnLlvKjrg+AGl74dDcNzW2u50SQO7hz8s9OSyImSpO/Ut3RG6ZUwGttM5RW1XUc+W8qr2FrxVHqGlsAiImMYGx6Uke9f3xmCjnpSdrto8KK3hmrQk5bm2H3oWOumn9NxxnAkWPNAERGCNlpiYzLSCHfdeSfMziZxFgdsFWFJk30KiwYYyg/cryj3r+p3JZ+quuaOrZJT45jVFoCo1MTGZWW2PGYlqQtnyq46Xj0KiyICFn9+pDVrw9Xj08HbPKvrG1kU1kN2w/UsrOqjp1V9SxaX95R+gFIio1iZFoio1ITGJWayOi0REalJjJsQB+i9eKvCnKa6FVIExEGJccxKC+OK/IGdSxv/wAoqayzyb+yjpKqOj4sOcji9eUd20VFCMMG9OmU/Ee5PhCSdBJ2FSQ00auw1PEBkBzHRaMHdlpX29BMaVX9iQ+BqjpKKut4e1slLW0nSp2DkmNPJH+3DwK981cFGk30SnWRFBd9UtsnQHNrG7sPHuuU/HdW1bO4SxkoMTaqowQ0Kq39QyCBYQMStAykHKGJXikvRUdGMDrNHrm7ay8DtZd/OspAOw+y+LPOZaChA/owOjWR7EGJZKcldZwFxMdoK6jyH030Sp0l9zLQhd2UgdrPANrLQSu3VXYM/yACWf3iXR8ANvlnuz5Q9DqA8gVN9Er5UXdloKaWNnYdrKf4gE3+xZW1lFTW8UHJwY4hIMC2g2YPSuw4k8hOSyI7LZF+CTG9/auoIKaJXikHxERFMGZQEmMGJXVa3tLaxt7Dxyk+UEtJVR0lB+oorqzjxU/3cry5tWO7AQkxNvG7lYCy0xJJ1fsBlAea6JUKIFGREYwYmMCIgQlc5ba8rc1QUXOc4kp7DaD4gD0L+GdhBbUNJy4EJ8dF2fKP6zpA+5lARkq8jgQaxvTOWKWCmDGGqtpGiivrOs4C2stBB+tP3BHcJyayI+mPdt0RnNE3nrSkWPonxOiIoCFA74xVKkSJCGnJcaR5uB/gUH1TR/2/+IC9CNz1hjD7GrYUNDAxltQkt6/ELo9JsaTER2tpKAhpolcqRPVPiGHqiP5MHdG/0/Kjrk6g/TUNVNU1UlXr9lXXSGlVPVV1jTS1tJ30mtGR0inxt38QDOzygZCaFEufGE0vgUL/JZQKM8lx0Uwc0rfznHBdGGM42tDS6QOg6wdC+ZEGNpTVcLCukTYPFeCEmMiTPxC6fkgkxZKWFEekXj/wK030SqmTiAgp8dGkxEefdINYV61thkP1Td1+IFTVNrB9fy0f1B2k5njzSc+PihDSU+LI6BtPVt94MvvFk9k3ngy373VugbOjiV4pdVYiI6Tj6Px0GppbOdj+oVDbyIGjDeyrOU754eOUHznOx6UH2X+04aQzhIGJMTbxt3+5fRhk9YvXawenoYleKdVr4qIjO5J1d1pa29h/tKEj+VccsY9lh4+z/UAtq7ZXdkwt2S4hJpLMfvEnPgz6df5QCPfykCZ6pVRAiYqM6JhXwBNjbKmo/MiJMwH37zfsPcLhY51LRFERwuC+cWSk2MTfXiJq/2DICPHykCZ6pVRQEREGJMYyIDGWCVl9PW5T39hCxZHjlLWfEbidHXy88+TykAhk9o1nZKqda6D9cVRqaMw+poleKRVyEmLtHcLZXYaYaNfc2sb+moaOstCeQ8coraqntLqOtbsOcazpxHATibFRjExNYORAm/hHpiYyMtXevRwsZwFeJXoRuRp4CogEfm+M+XmX9eJaPxs4BtxqjFnvWrcLqAVagZbu7txSSqneEh0ZwZD+fRjS/+TykDGG/UcbOkYdbX9cs+sw/yis6Niu/SxglCvxB/JZwGkTvYhEAvOBK4EyYI2ILDHGbHXbbBaQ7fo6H3jG9djuMmNMtc+iVkopPxERBqfEMzgl/qS7jY81tfB5df1JHwKffn6o06Bzns4CRqUlMHyAM2cB3hzRTwVKjDGlACLyIjAHcE/0c4DnjR0452MR6Ssig40x+3wesVJKOaRPTBTjMlIYl5HSaXn7WcDOSlv+2VlZR2l1/WnPAtwf/XkW4E2izwT2uv1cRuej9e62yQT2AQZ4S0QM8L/GmAWediIidwB3AAwdOtSr4JVSKhC4nwV8IdvzWcDOqnpKq+o6Hj2dBeQOTuLlf73A5wnfm0TvaY9db3g+1TYXGWMqRCQNWC4i24wxq0/a2H4ALAA7eqUXcSmlVMDr7iygra3rtYA6Glva/HJU702iL6PzqBhZQIW32xhj2h8rReTv2FLQSYleKaXCSUSEkOHq4e96FuDzfXmxzRogW0RGiEgMMA9Y0mWbJcAtYk0Daowx+0QkQUSSAEQkAbgK2OzD+JVSSp3GaY/ojTEtInI38Ca2vfI5Y8wWEbnTtf5ZYCm2tbIE2155m+vpg4C/u05FooAXjDHLfP5bKKWU6pbOMKWUUiHgVDNM6fxhSikV4jTRK6VUiNNEr5RSIU4TvVJKhThN9EopFeICsutGRKqA3Wf49IGADqBm6XvRmb4fnen7cUIovBfDjDGpnlYEZKI/GyKyVodCtvS96Ezfj870/Tgh1N8LLd0opVSI00SvlFIhLhQTvcdhkMOUvhed6fvRmb4fJ4T0exFyNXqllFKdheIRvVJKKTea6JVSKsSFTKIXkatFZLuIlIjIw07H4yQRGSIiq0SkSES2iMi9TsfkNBGJFJHPROQ1p2NxmmtO51dEZJvrb+QCp2Nykojc7/p/sllE/iYicU7H5GshkehFJBKYD8wC8oAbRSTP2agc1QL8uzEmF5gGfDvM3w+Ae4Eip4MIEE8By4wxOcA5hPH7IiKZwHeAKcaY8dg5N+Y5G5XvhUSix05PWGKMKTXGNAEvAnMcjskxxph9xpj1ru9rsf+RM52NyjkikgV8Efi907E4TUSSgenAHwCMMU3GmCPORuW4KCBeRKKAPpw8VWrQC5VEnwnsdfu5jDBObO5EZDhwLvCJs5E46kngu0Cb04EEgJFAFfBHVynr965pPsOSMaYc+BWwB9iHnQb1LWej8r1QSfSepk0P+75REUkEFgH3GWOOOh2PE0TkGqDSGLPO6VgCRBQwCXjGGHMuUA+E7TUtEemHPfsfAWQACSLyNWej8r1QSfRlwBC3n7MIwdOvnhCRaGySX2iMWex0PA66CCgQkV3Ykt7lIvJXZ0NyVBlQZoxpP8N7BZv4w9UVwOfGmCpjTDOwGLjQ4Zh8LlQS/RogW0RGiEgM9mLKEodjcozY2dj/ABQZY37tdDxOMsY8YozJMsYMx/5dvG2MCbkjNm8ZY/YDe0VkrGvRDGCrgyE5bQ8wTUT6uP7fzCAEL05HOR2ALxhjWkTkbuBN7FXz54wxWxwOy0kXATcDm0Sk0LXse8aYpQ7GpALHPcBC10FRKXCbw/E4xhjziYi8AqzHdqt9RggOh6BDICilVIgLldKNUkqpbmiiV0qpEKeJXimlQpwmeqWUCnGa6JVSKsRpoldKqRCniV4ppULc/wcwz5kwsaFjhAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot some data\n",
    "plt.plot(r.history['loss'], label='loss')\n",
    "plt.plot(r.history['val_loss'], label='val_loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxdZZ348c83e5N0SZp0oWmb0BaaLilLWgsKqEUtiFYRC4yKK/xwQcSfMyI6g/MbdVCZn8OM/GA6yoxokcEKI6MIKltdWBpK2iRtgdItSbc0+9Ks9/v74zk3uUnT5ia5ybnL9/165XXvPetzbpLne87zfc5zRFUxxhiTeJL8LoAxxhh/WAAwxpgEZQHAGGMSlAUAY4xJUBYAjDEmQaX4XYDRyMvL08LCQr+LYYwxMeWVV145oar5Q6fHVAAoLCykrKzM72IYY0xMEZGDw023JiBjjElQFgCMMSZBWQAwxpgEZQHAGGMSlAUAY4xJUBYAjDEmQVkAMMaYBBVWABCR9SLymojsFZHbh5mfIyKPichOEXlZRFaEzLtVRCpFpEpEvjTMul8RERWRvPEdijFmXFTh9afgT/8MFVug+mVoOQKBgN8lMxNkxBvBRCQZuBd4F1ADbBORx1V1V8hidwDlqvpBEVnqLb/OCwQ3AmuAbuBJEfmNqr7hbXu+t91DkTyoU9S8Au3HYVYxTF8ASXbhY0w/VXjzGXjmW3B4+6nzk1Jh+jyYPh9mLIDpBd77+e51egGkpE9+uc24hXMn8Bpgr6ruAxCRh4ENQGgAWAb8I4Cq7hGRQhGZDRQDL6pqh7fu88AHge956/0A+BvgVxE4ltN75QF49WfufWoWzFrqgsGsZQOv2bNBZEKLYaJQXw+0HnEVWSL+/g/8CZ75Nhz6izs5ev8PofgqaD0KTdXQfMh7rYHmanjzWfd9MeRBUtmzQ4JCgdtWaICYMsOXw4t5gQB01LvvPKcQMqZFdPPhBIB5QHXI5xrgLUOW2QFcDfxJRNYAC4ECoBL4tojMBE4CVwJlACLyfqBWVXfIGf7xROQm4CaABQsWhFHcYbznH+GCT8DxXXB8t3t9/XcDQQFgSg7kF3sBISQ4ZOaObZ8mejXsc2e8bz4L+7dCVwvMuxAu+jwUb4DkmBohZWyqt8Gz34J9z8HUufDef4Lzb4CUNDd/So77+x9Obze01A4EhWCgaK6BIzthzxPQ1zV4nfRpQwJE8L13RZE9O7GuzFWhq9VV7K1HXMBtOexeQ6e1HoVAj1vno7+ExZdHtBjh/KUPVzsPfY7kXcA9IlIOVACvAr2qultEvgv8HmjDBYpeEckEvg68e6Sdq+omYBNAaWnp2J5fmTEN5q92P6HaT3gBYfdAcKjYAl3NA8tkzzn1aiH/XEjPHlNRjA86W1xF/+Yz7qdxv5s+fQGsuNqdWW1/ELZ8ylVMb7kZLvgYZEz3tdgT4sgOePY78PqTkJkH7/kOlH4KUqeEv42UNMgtcj/DCQSgvc4LEMErCO8qoqkaDr0Anc2D10lOg2nzXDCYscD9HrLyIC0b0qe6/7e04Ks3LS07OoNGb9cIlfoRl1vpaT913fRpLiBPnQOFb3OvU+e6nzklES+qjPRMYBG5CPimqr7H+/w1AFX9x9MsL8B+oERVW4bM+w7uCuKPwNNAhzerADgMrFHVo6crS2lpqU74YHCq7pcWGhSO74K616D35MByMxZ6QWHpQHDIO8faQqNBoA8Ol8ObT7sKv/pl0D7X/Fd0CSxaB4veCTMXDTT7BAKuUnzhXjj4J1fZXHADrL3ZVUix7vhuV/HvfhwyZsBbvwhr/pd/JzKdLSFXEIcGB4jmaldRnnKeOYzUrMFBIRgY+qdlu0p10LShy3mBZaT/3UCfC2zBCrz1yJAzeO/9yYZT101OH6jMp80dqNSDlf20s9xV0AT9PkTkFVUtPWV6GAEgBXgdWAfUAtuAv1LVqpBlZgAdqtotIjcCl6jqDd68Wap6XEQWAL8DLlLVxiH7OACUquqJM5VlUgLA6QT6oPGA+0eq2z1w5XDidQj0umUk2VUqQ68YcooSo1nBT801sNer8Pc9B51Nbvrc81xlv3gdFKwZaOI4k9rt8OL/g8pHAYVlG+CiL0DBKf8/0a/+TXjuLqj4havwLvo8XPS56L+66e12v8OuVuhug64277U1jGmt7jU4rbczvH0mp50aFNKy4GSTq+DbjrkTiVCSBFmzQir1OTD1rFMr+yk5vuaYxhwAvJWvBP4ZSAYeUNVvi8jNAKp6v3eV8CDQh0sOfzpYyYvIH4GZQA/wZVV9epjtHyDaA8Dp9HZDw5shVwveFUPDfvrPYJLTIf8cFwxKPwUL1vpa5LjQ3Q4H/jxwln/idTd96lxX4S96J5z9dteMMFbNNfDyJij7T9csOP8tLhAsfS8kJUfgICZQ0yF4/ntQ/pA7s11zE7z11sTMafX1hBFEWkLehyzX3eaC5dCz9WAFnzUrJk7uxhUAokVUBoDT6e6AE68NbkqqKXOXnrfujM62y2gWCMCxClfZ730aql+Cvm5IyYCFbx2o9GcVR/5Mq6sVXt3srgqaDrrmv7Wfg/M/4n6f0aTlCPzxbnjlJ+7sdPWn4W23QfYsv0tmfGQBIBrsfAQevRE+8QQUvtXv0kS/1mMDidt9z7r2V4BZy2GxV+EvuGh0CczxCPTBnl+7PEH1S5A+HUo/4drSp8+bnDKcTlsd/PmfYduPXJPkBTfAJV/xv1wmKpwuAET/tUs8Wfpel7SqeMQCwHB6Ol1/9GAXzWOVbnpm3sAZ/qJ3uMtvPyQlu3zAsg2uG+WL98Jf/tUFhOUfdO3rZ50/uWXqaIAXfggv3u86Kay6Hi77G9ezyZgRWACYTGlZLghUPQZXfM96DKlC3Z6B5O3BP7uEXVKqy5Nc/k1X6c9eGX1NZvNXw/z/hMaD8NK/uW6kFb+AhW9zgeCc9RNb5s4WePE+V/l3tcKKD8Hbb4e8JRO3TxN3rAlosr3xB9j8Ibh2s7vjMlF1NMAD612eBFwX2kXvdF00C9/qguUkCQSUpKRx5g06m2H7T+Gl+103xtxFrrfNqusjeyzd7fDyv7vmnpONsPQqeMcdMHt55PZh4o41AUWLs98OWfmw878SOwBUbHGV/3v+0X0PE9DXvqcvwIm2Lo61dHG0uZPjrZ0ca+nkaHMXx1s7OdrsPrd19TI/N5PF+dksnp3NkllTWTIrm0WzsslOD/NfJGM6XPwFdxPZ7l/BX34Iv/nfbnyd0k+5Xjjjabrq6YRX/hP++E9uXKvF73IV/7wLxr5Nj6rS2NHD/hPtHDjRTmtnD5npKWSlpZCZnkx2egqZacn9n7PSUpiSmjz+oGl8Z1cAfvjtV6HsP+ArryfuGCn/dhloAG7+46hXDVZYR5s7OdbayXGvUu9/39LJsZYuTrR1MfTPOzlJmDU1nVnTMpgzLZ3Z0zLITk/hYEMHe4+1sf9EO919A6NfnjU9g8WzXUBYPCu7/3VG5gj3E6i6RPFf/hX2/AaSUmDlh91VwZyV4R9sbzeU/wy23u2GXyi6FN7xDVgwdDSWkbV09nDgRLtX0Xew/0Qb++s7OHCineaTPaPeXmZaMplpKWSlu9fs9MGfs9KSvUAyMD0rGFjS3PvQ18y0FJLHGFRUlYBCX0AJqNIbUPc+oPSpez/oR928/uW8dQJDlgm+7+lzr72BAL3e+55AwE3r86b3v1f6vOXctED/ftx2AvQElL7gfG87PX3e9kK20xsy7fvXlPCWs2eO6fuxK4BosnKjayrY/bjrrZFojlXBkXJY/91TZrV39XoVePCn65T3x1u6BlXSQblZacyelsHsaeksnzud2dPd+znTMpg9LYNZ09KZmZV+xkqmty/AoYYO3jjexl7v543jrWx+qZ7OnoF95k9NZ3F+NktmB4PCVBbPyiYvOw0RcV1RF6x1Pw37XJL21Z/Bjoeg6DJ3P8Hiy0+fJ+jrdZ0FnrvLdT2d/xb4wH1w9mVn/GpPdvdxoN6dye/zzuj3n2jnQH07J9q6By07b8YUCvMyuapkLkV5WRTlZVGYl8WMKal0dPfR0d1He3cv7V29tHf10dHdS3t3Hx1dQ167B+Y3neyhtulk/7z2rl56A+GfZGakJvVfaSSLeBU1XiWJq6j7Av2VfWhFHU1SkoTkJCE1OYnkJCElSUhJFlKSkkhJloFp3uf+90lJZKQGl08atJ2pGakRL6ddAfhBFf71QndDySd+7XdpJl3nr28nbfuPuO+C37C3Pd01y3gVe1tX7ynLZ6enMGtaOrOnZjBnesag97O9s/j8qemkp0zczVmBgFLbdLI/ILxxrI03jrfx5vE2WkPKPCMzdVBAWDLLBYk50zKQzibXjPPSv7khA/LOdVcEJdcOdGUNBKDqUVfx178Bc1fBO//WBQvv/oau3j6qGzrYf6JjUEV/oL6dI82D73rNn5ruKveZrnIPVvQLZ2aSkTo5N7N19wbOGDzaunrp6HLBpsMLGu1dvQTUVaRJSUKyuNdghZgkrkJNEiE5CZKTkkgOfZ+EN08GbSM5achPyHaHWyZJhNT+Cjuksg6tnJOSSO6vxN20Mw1w6Qe7DyDaPHeX+7mtKu77atc2nWTb/ga2HWhg+/7jPNj8CcoC53JL35eZPW2gIp8VWqlPzfDO4DPCb4f3gapyrKWrPyjsrWtj77E2Xj/eSlPHQLNKdnoKi7yAcG5eOhd3/ZElb/4naXWVkDkTVn/GDTK49Z/geBWaX0zd6q+wa9ol/c00++tds01t40lCT3hzMlMHKveQir4wLyuqvzszeSwARJv6N+FfL4B3/R93i36cCASU14+3su1AI9v2N1B2oIHD3llpdnoKn8rfw5dP/B173vHvLLzoQ0xJi/IhFcZIValv7w4JCq28cdxdNdS1BodKVi5N3cPnM57iLT0vA3A0pYB/T7mWn7ZeQHffwFnk1PQUCvNCz+IzKZzp3o+YjzAJzwJANPr3da7f+2f/7HdJxqyrt4+KmmZX4R9wFX5Lp2sSmTU1ndVFuaxemENpYS7Fc6eR/MhHXXL0y7shOfJtmrGguaOHvXUDzUh7j7fRffQ1pnfWcChnLQvzpw1qrimcmTWQWzBmDCwJHI1KroXf/rVLisZIP+7mkz1sPxQ8u2+kvKaJ7l6XHF2Un8WVK+dSWpjLmsJc5udOGVxptZ9wQy6/5eaErfwBpmemcuHCXC5cGDow2xrfymMSlwUAPy3/IDx5uxsj6F1/73dphnW0uZOXvTP7bQca2XO0BfWSc8vnTeeGtQtZXZRL6cIcZmaPcGdzxRY3Ts15fzU5hTfGnJEFAD9l57tx6iu2wLo7fR/uQFV5s66Nl/c3UnaggZcPNFDT6B6Ck5mWzAULcvjSunNYXZjDeQtmkJk2yj+f8s1ufP4YudoxJt5ZAPDbyo3wxmfcIGiFb5vUXXf3Bqg83Nx/dl92oIFGr+dKXnYaqwtz+eRbi1hTmEvx3KmkJI8jQB2tgKM74YrvR6j0xpjxsgDgt6VXuhFCdz4yYQGguzfA4aaTVDd2UN1wkkMNHZRXN1Je3dR/c1NRXhaXF89mdWEuq4tyKZyZGdmkY/lDbpC3lddEbpvGmHGxAOC3tCw3Fs6u/4Yrvz+mEUL7Asqxlk6qGzqobjzpvXZQ03iSmoYOjrR0DhoSISVJKJ47jevXLGBNYS4XFuYwa2pGBA9qiN5uN/bRuVck5hOpjIlSFgCiQclGV0G+8Tsoft8ps1WVhvbuQZV7dcNJaho7qG7ooLbpJD19AzW8CMyemsH83CmsPXsmBbmZzM+ZwvzcTObnZjJnWsaYx1wZk72/h456OO8jk7dPY8yILABEg6K3E8jMp+3lzbwYWN1f0deEVPTt3YMfRp2Tmcr83EyWnzWd96yYw/wcV7nPz5nCvJwpEzoswqiVP+Senbr4cr9LYowJYQFgEjV39FBe0zTQRNMQbJfv4JbuC/lI++/5yu6ttJBFVlpy/xn7xYtnDlTwuVMoyMmMnVv82+pc3/+1n42Jh2cbk0jsP3IS3fyzV3hhXz0AaclJzMuZQkHOFFasnMu05I+Qvv1JfnN5PVkXfYCczNT4uPOz4heu7/8q6/tvTLSxADBJevsCvFrdyNXnz+Ov15/L7KkZgx+ooSvg4GLm1/wGsj7rX0Ejrfwh95zc2cv8LokxZogoe9Bq/Hqzrp3OngBvW5LH3OlTTn2akoi7J+DAH6G5xp9CRtqRnXCswpK/xkQpCwCTpKK2GYCV86affqGSD3sLb5mEEk2C8s2QnOYeWG6MiToWACZJZW0zmWnJnJ2fffqFcs+GgtXuprBY19vtjuPcK63vvzFRygLAJNlZ08Tys6aN3P++5Fo4XgVHKyenYBPljafgZIM1/xgTxSwATILevgC7jrSw4kzNP0HLPwiS7J4HG8vKH4LsObDonX6XxBhzGhYAJkEwAXzG9v+grDx3w1TFFvd82FjUdhxefwpWXWt9/42JYmEFABFZLyKvicheEbl9mPk5IvKYiOwUkZdFZEXIvFtFpFJEqkTkSyHTvy8ie7x1HhORGZE5pOgTVgI4VMlGaKmFgzH6pLCdj4D2Wd9/Y6LciAFARJKBe4ErgGXA9SIytFP3HUC5qpYANwD3eOuuAG7EPe5oFXCViCzx1vk9sMJb53Xga+M/nOgUVgI41LlXQlp2bDYDqbreP/MuhFlL/S6NMeYMwrkCWAPsVdV9qtoNPAxsGLLMMuBpAFXdAxSKyGygGHhRVTtUtRd4Hvigt9zvvGkALwIF4z6aKFVR28yyuWEkgIPSMmHpVVD1K+jpnNjCRdqRHXB8lz31y5gYEE4AmAdUh3yu8aaF2gFcDSAia4CFuAq9ErhURGaKSCZwJTB/mH18CvjtcDsXkZtEpExEyurq6sIobnTpCyi7DoeZAA5VshG6mt0IobGk/CFITre+/8bEgHACwHCnrTrk811AjoiUA7cArwK9qrob+C6uuedJXKDoDV1RRL7uTds83M5VdZOqlqpqaX5+fhjFjS5v1rVxsqePkoJRBoCiy9wImjv/a2IKNhF6u1yz1dL3wpQcv0tjjBlBOF00ahh81l4AHA5dQFVbgE8CiBvBbL/3g6r+GPixN+873vbwPn8cuApYp6pDg0pcqKgZZQI4KDnFPT1r24/gZGNsVKivP+nKan3/jYkJ4VwBbAOWiEiRiKQB1wGPhy4gIjO8eQCfAbZ6QQERmeW9LsA1E/3c+7we+CrwflXtiMTBRKOK0SaAQ638MPR1w65fRb5gE6H8IZg6Fxa9w++SGGPCMGIA8BK1XwCeAnYDj6hqlYjcLCI3e4sVA1UisgfXW+jWkE38UkR2Af8DfF5VG73pPwSmAr8XkXIRuT8yhxRdRp0ADnXW+TBzCez8ReQLFmmtx+CN37s7mZOi6GE0xpjTCusuHVV9AnhiyLT7Q96/ACwZup4375LTTF8cfjFjUzABfO3q4fLeYRBxyeBnvw1N1TBjjNuZDBVe339r/jEmZtidwBMomAAedft/qJXeCKGVUTxCqCq8utkNZJd/jt+lMcaEyQLABOpPAI+2B1Co3CIoWBPdI4QefhXqdlvff2NijAWACVRR28yU1GQWjSUBHKpko7u5KlpHCA32/V9+td8lMcaMggWACVRZ28yycIaAHsnyqyEpJTrvCejtcs/9Lb4KpsTtcE7GxCULABOkL6BUHW4ZX/t/UNbM6B0h9LUnoLPJkr/GxCALABNkXyQSwKFWfhhaD8PBP0Vme5FS/hBMPQvOfrvfJTHGjJIFgAmyMxIJ4FDBEUKjKRncehT2/gFWXWd9/42JQRYAJkjEEsBBaZlQ/D53V3C0jBC6879AA9b8Y0yMsgAwQSKWAA5VshG6Wtzzdv0W7Ps//y2QF/f39BkTlywATICIJoBDFV0G2bOjoxmodjuceM36/hsTwywATIBgAnjUzwAYSVIyrLjGPSOgoyGy2x6t8s2QkuEeYm+MiUkWACbAqJ8BPBolUTBCaE+nG5qi+H2QMQHHaIyZFBYAJkBFbTMZqUksys+K/Mbnngd557ibr/zy2hPQ2WzNP8bEOAsAE6CytpnlZ00nJXkCvl4RWLkRDv4Zmg5FfvvhKN8M0wpcTsIYE7MsAERYX0CprJ2ABHColde41wofRghtOQxvPmN9/42JAxYAImzCEsChcotc98udj7jumJOpv++/Nf8YE+ssAETYhCaAQ5VsdEMwH5vEEUJV3dAP89fCzEWTt19jzISwABBhE5oADrXsg5M/QmhNGZx43c7+jYkTFgAirNJ7BvCEJIBDZc2Exe+Cil9CoG9i9xVUvhlSpljff2PihAWACJqwO4BPp8QbIfTAJIwQ2nMSKh+FZe+HjGkTvz9jzISzABBB+0+00dE9wQngUOdcAWlT3QPZJ9qe30CX9f03Jp5YAIigYAK4pGCSnozVP0Lo4xM/Qmj5QzB9PhReOrH7McZMGgsAEVRR0zI5CeBQwRFCX39y4vbRXOv1/b8ekuxPxph4Yf/NEVRR2zQ5CeBQRZdC9pyJHSF058OAwnnXT9w+jDGTzgJAhEx6AjgoKdndGTxRI4QG+/4vuBhyz4789o0xvrEAECGTngAOtfLDEOiBXf8d+W3XbIP6vZb8NSYOWQCIkP47gCP1DODRmLsK8s6FnRMwQuirP4PUTFj+gchv2xjjKwsAERJMAC+O1DOAR0PE3RNw6C+RHSG0uwOqHoNlGyB9auS2a4yJCmEFABFZLyKvicheEbl9mPk5IvKYiOwUkZdFZEXIvFtFpFJEqkTkSyHTc0Xk9yLyhveaE5lD8kdlbTPFk50ADrXyw+41ks8J2PMb18PImn+MiUsj1lYikgzcC1wBLAOuF5FlQxa7AyhX1RLgBuAeb90VwI3AGmAVcJWILPHWuR14WlWXAE97n2NSIKBUHW6e/ARwqJxCN0hbJEcILd8M0xfAwrdFZnvGmKgSzunqGmCvqu5T1W7gYWDDkGWW4SpxVHUPUCgis4Fi4EVV7VDVXuB5IDiQzAbgJ977nwAx28i870Q77d19/gYA8EYI3QNHK8a/raZq2Pec6/ppff+NiUvh/GfPA6pDPtd400LtAK4GEJE1wEKgAKgELhWRmSKSCVwJzPfWma2qRwC811nD7VxEbhKRMhEpq6urC++oJllFbRPgUwI41HJvhNBIDA0R7Pu/yvr+GxOvwgkAMsy0oW0MdwE5IlIO3AK8CvSq6m7gu8DvgSdxgaJ3NAVU1U2qWqqqpfn5+aNZddL4mgAOlZkLS97tnhQ2nhFCg33/F77NPXzGGBOXwgkANQyctYM7sz8cuoCqtqjqJ1X1PFwOIB/Y7837sapeoKqXAg3AG95qx0RkLoD3enxcR+Ij3xPAoVZ+GFqPjG+E0OqXoGGfJX+NiXPh1FjbgCUiUiQiacB1wOOhC4jIDG8ewGeArara4s2b5b0uwDUT/dxb7nHg4977jwO/Gs+B+CUqEsChzvVGCB3P0BDlmyE1y3X/NMbErREDgJe8/QLwFLAbeERVq0TkZhG52VusGKgSkT243kK3hmzilyKyC/gf4POq2uhNvwt4l4i8AbzL+xxzgglgX+4AHk7qFDdm/+7H3Rj+o9XdDpXBvv8+N2kZYyZUSjgLqeoTwBNDpt0f8v4FYMnQ9bx5l5xmej2wLuySRqnKyXoG8GiUbHRn8a8/Ofqnd+3+NXS3wvkfmZiyGWOiRhQ0Wse2itpm0lOSWDIris6WCy/xRggdw01h5ZthxkI3+JsxJq5ZABinitpmlp0VJQngoLGOENp0CPZvdclf6/tvTNyz//JxCASUqtooSgCHKtk4+hFCdwT7/l83YcUyxkQPCwDjEHUJ4FBzSiB/afi9gVRd80/hJW5YCWNM3LMAMA5RmQAOEnH3BBx6ARoPjrz8oReg8QCcZ8lfYxKFBYBxiMoEcKjRjBBavhnSsl0XUmNMQrAAMA4V0XQH8HByFsKCi0YeIbS7Har+G5Z9ANIm8YH2xhhfRWnNFf0CAWWXH88AHq2SjXDiNTi68/TL7Hocutts6AdjEowFgDHaX99OW1dv9AeAZR+ApNQzJ4PLN7vE70Lr+29MIrEAMEbBBHBU9gAKNdIIoY0H4cAfXfJXhhv41RgTrywAjFFFjZcAnh2lCeBQJR+GtqOuoh9qx8Pu1fr+G5NwLACM0U4vAZwarQngUOesh/RppzYDBQKu+afoUpixwJ+yGWN8EwO1V/SJmQRwUOoUKH6/S/aGjhB66C/QdND6/huToCwAjEHMJIBDlWx0o3y+9tuBaeUPuWcHFL/Pv3IZY3xjAWAMYiYBHKrwbTB17sBNYV1tru//cuv7b0yisgAwBhU1zaTFSgI4aOgIobsfh552a/4xJoFZABiDilhKAIdauRECvVD1GLy6GXKKYMFav0tljPFJjNVg/nPPAG5h5bxpfhdl9OashPxieOFeOPgn6/tvTIKzADBKB2IxARwk4u4JaHgTEOv7b0yCswAwShX9Q0DP8LkkYxQcIfTsy2DGfH/LYozxVVgPhTcDYjIBHGrGAnj/D+Gs8/0uiTHGZxYARilmE8ChLviY3yUwxkSBGK7FJl9MJ4CNMWYICwCjENMJYGOMGcICwChUxOIdwMYYcxoWAEahstYlgM+ZPdXvohhjzLhZABiFitpmiudMje0EsDHGeKwmC1MgoFTVtrCywJp/jDHxIawAICLrReQ1EdkrIrcPMz9HRB4TkZ0i8rKIrAiZd5uIVIlIpYj8XEQyvOnniciLIlIuImUisiZyhxV5Bxs6aLUEsDEmjowYAEQkGbgXuAJYBlwvIsuGLHYHUK6qJcANwD3euvOALwKlqroCSAaC4w98D/h7VT0P+Dvvc9TaWdMEWALYGBM/wrkCWAPsVdV9qtoNPAxsGLLMMuBpAFXdAxSKyGxvXgowRURSgEzgsDddgWCH+ukh06OSJYCNMfEmnAAwD6gO+VzjTQu1A7gawGvKWQgUqGotcDdwCDgCNKvq77x1vgR8X0SqvWW+NtzOReQmr4morK6uLryjmgCWADbGxJtwarPhxgvWIZ/vAnJEpBy4BXgV6BWRHNzVQhFwFpAlIh/11vkscJuqzpXVUhAAAA+0SURBVAduA3483M5VdZOqlqpqaX5+fhjFjbxgAtiaf4wx8SScAFADhA4bWcCQ5hpVbVHVT3rt+TcA+cB+4HJgv6rWqWoP8Chwsbfax73PAL/ANTVFJUsAG2PiUTgBYBuwRESKRCQNl8R9PHQBEZnhzQP4DLBVVVtwTT9rRSRTRARYB+z2ljsMXOa9fyfwxvgOZeLYHcDGmHg04migqtorIl8AnsL14nlAVatE5GZv/v1AMfCgiPQBu4BPe/NeEpEtwHagF9c0tMnb9I3APV5yuBO4KaJHFkGVtc2kJVsC2BgTX8IaDlpVnwCeGDLt/pD3LwBLTrPuncCdw0z/E3DhaArrl4qaZornTiUtxRLAxpj4YTXaCAIBpbK22Zp/jDFxxwLACCwBbIyJVxYARmAJYGNMvLIAMAJLABtj4pUFgBFU1DSz1BLAxpg4ZLXaGagqlYctAWyMiU8WAM7gYH0HrZ2WADbGxCcLAGcQTABbADDGxCMLAGdgCWBjTDyzAHAGOy0BbIyJY1aznYYlgI0x8c4CwGlYAtgYE+8sAJyGJYCNMfHOAsBpWALYGBPvLACcRkVtM+fOsQSwMSZ+We02DFUbAtoYE/8sAAzjUEMHLZ29lBRYADDGxC8LAMPYWWMJYGNM/LMAMAxLABtjEoEFgGFYAtgYkwishhvCEsDGmERhAWCIYALY2v+NMfHOAsAQdgewMSZRWAAYoqK2mdRk4Zw52X4XxRhjJpQFgCEqvQRwekqy30UxxpgJZQEghKpSUdPMynkz/C6KMcZMOAsAISwBbIxJJBYAQlgC2BiTSMIKACKyXkReE5G9InL7MPNzROQxEdkpIi+LyIqQebeJSJWIVIrIz0UkI2TeLd52q0Tke5E5pLGzBLAxJpGMGABEJBm4F7gCWAZcLyLLhix2B1CuqiXADcA93rrzgC8Cpaq6AkgGrvPmvQPYAJSo6nLg7ogc0ThYAtgYk0jCuQJYA+xV1X2q2g08jKu4Qy0DngZQ1T1AoYjM9ualAFNEJAXIBA570z8L3KWqXd56x8d1JOPk7gBuseYfY0zCCCcAzAOqQz7XeNNC7QCuBhCRNcBCoEBVa3Fn9oeAI0Czqv7OW+cc4BIReUlEnheR1cPtXERuEpEyESmrq6sL97hGrbrhJM0ne2wICGNMwggnAMgw03TI57uAHBEpB24BXgV6RSQHd7VQBJwFZInIR711UoAcYC3w18AjInLKvlR1k6qWqmppfn5+OMc0JpYANsYkmpQwlqkB5od8LmCgGQcAVW0BPgngVeL7vZ/3APtVtc6b9yhwMfAzb7uPqqoCL4tIAMgDJu40/wyCCeBz59gQ0MaYxBDOFcA2YImIFIlIGi6J+3joAiIyw5sH8BlgqxcUDgFrRSTTCwzrgN3ecv8NvNNb/xwgDTgx3gMaK0sAG2MSzYhXAKraKyJfAJ7C9eJ5QFWrRORmb/79QDHwoIj0AbuAT3vzXhKRLcB2oBfXNLTJ2/QDwAMiUgl0Ax/3rgYmnapSUdvMlSvn+LF7Y4zxRThNQKjqE8ATQ6bdH/L+BWDJada9E7hzmOndwEdPXWPyWQLYGJOI7E5gLAFsjElMFgCwBLAxJjFZAMAlgM+ZbQlgY0xiSfgAEEwAW/OPMSbRJHwAqGm0BLAxJjElfAAIJoBLCiwAGGMSS8IHgJ01lgA2xiSmhA8AlgA2xiSqhA4AlgA2xiSyhA4AlgA2xiSyhA4AdgewMSaRJXwASEmyBLAxJjEldAAIJoAzUi0BbIxJPGGNBhqPggng9cttCGhjYkVPTw81NTV0dnb6XZSolJGRQUFBAampqWEtn7ABoKbxJE0dlgA2JpbU1NQwdepUCgsLGeYJsglNVamvr6empoaioqKw1knYJiBLABsTezo7O5k5c6ZV/sMQEWbOnDmqq6OEDgCWADYm9ljlf3qj/W4SNgBYAtgYk+gSMgDYHcDGGJOgAaA/AWwjgBpjElhC9gKqtASwMTHv7/+nil2HWyK6zWVnTePO9y0/4zIf+MAHqK6uprOzk1tvvZWbbrqJJ598kjvuuIO+vj7y8vJ4+umnaWtr45ZbbqGsrAwR4c477+RDH/pQRMs7XgkZAIIJ4KWWADbGjNIDDzxAbm4uJ0+eZPXq1WzYsIEbb7yRrVu3UlRURENDAwD/8A//wPTp06moqACgsbHRz2IPK2EDgCWAjYltI52pT5R/+Zd/4bHHHgOgurqaTZs2cemll/b3vc/NzQXgD3/4Aw8//HD/ejk5OZNf2BEkXA7AEsDGmLF67rnn+MMf/sALL7zAjh07OP/881m1atWw3S9VNeq7rCZcALAEsDFmrJqbm8nJySEzM5M9e/bw4osv0tXVxfPPP8/+/fsB+puA3v3ud/PDH/6wf91obAJKuABgCWBjzFitX7+e3t5eSkpK+Nu//VvWrl1Lfn4+mzZt4uqrr2bVqlVce+21AHzjG9+gsbGRFStWsGrVKp599lmfS3+qhMsBWALYGDNW6enp/Pa3vx123hVXXDHoc3Z2Nj/5yU8mo1hjlnBXABW1zSyxBLAxxoQXAERkvYi8JiJ7ReT2YebniMhjIrJTRF4WkRUh824TkSoRqRSRn4tIxpB1vyIiKiJ54z+cM1NVKmubWTlv2kTvyhhjot6IAUBEkoF7gSuAZcD1IrJsyGJ3AOWqWgLcANzjrTsP+CJQqqorgGTgupBtzwfeBRwa/6GMrLbpJI0dPdb+b4wxhHcFsAbYq6r7VLUbeBjYMGSZZcDTAKq6BygUkdnevBRgioikAJnA4ZD1fgD8DaBjP4TwBRPA9gwAY4wJLwDMA6pDPtd400LtAK4GEJE1wEKgQFVrgbtxZ/hHgGZV/Z233PuBWlXdMa4jGIVgArh4rjUBGWNMOAFguDsZhp6x3wXkiEg5cAvwKtArIjm4q4Ui4CwgS0Q+KiKZwNeBvxtx5yI3iUiZiJTV1dWFUdzT21ljCWBjjAkKJwDUAPNDPhcwuBkHVW1R1U+q6nm4HEA+sB+4HNivqnWq2gM8ClwMLMIFhR0icsDb5nYROeUBvaq6SVVLVbU0Pz9/1AcYsh1LABtjTIhwAsA2YImIFIlIGi6J+3joAiIyw5sH8Blgq6q24Jp+1opIprh7otcBu1W1QlVnqWqhqhbigswFqno0Qsd1CksAG2MmW3Z2tt9FOKMRbwRT1V4R+QLwFK4XzwOqWiUiN3vz7weKgQdFpA/YBXzam/eSiGwBtgO9uKahTRNyJCOwBLAxcea3t8PRishuc85KuOKuyG4zioV1J7CqPgE8MWTa/SHvXwCWnGbdO4E7R9h+YTjlGI+K2maSLQFsjBmHr371qyxcuJDPfe5zAHzzm99ERNi6dSuNjY309PTwrW99iw0bhnaUPFVbWxsbNmwYdr0HH3yQu+++GxGhpKSEn/70pxw7doybb76Zffv2AXDfffdx8cUXj++AVDVmfi688EIdq4/9+CV9zw+eH/P6xhj/7dq1y9f9b9++XS+99NL+z8XFxXrw4EFtbm5WVdW6ujpdtGiRBgIBVVXNyso67bZ6enqGXa+yslLPOeccraurU1XV+vp6VVXduHGj/uAHP1BV1d7eXm1qahp2u8N9R0CZDlOnJsRYQOolgNctneV3UYwxMez888/n+PHjHD58mLq6OnJycpg7dy633XYbW7duJSkpidraWo4dO8acOaf0aRlEVbnjjjtOWe+ZZ57hmmuuIS/PDY4QfL7AM888w4MPPghAcnIy06ePvzk7IQLA4eZOGtq7KbEhoI0x43TNNdewZcsWjh49ynXXXcfmzZupq6vjlVdeITU1lcLCQjo7O0fczunW00l8jkBCDAZXUdMEWALYGDN+1113HQ8//DBbtmzhmmuuobm5mVmzZpGamsqzzz7LwYMHw9rO6dZbt24djzzyCPX19cDA8wXWrVvHfffdB0BfXx8tLeN/HnJiBABLABtjImT58uW0trYyb9485s6dy0c+8hHKysooLS1l8+bNLF26NKztnG695cuX8/Wvf53LLruMVatW8eUvfxmAe+65h2effZaVK1dy4YUXUlVVNe5jEZcfiA2lpaVaVlY26vX+a9shth9s4rvXlExAqYwxk2X37t0UFxf7XYyoNtx3JCKvqGrp0GUTIgdw7eoFXLt6gd/FMMaYqJIQAcAYY/xSUVHBxz72sUHT0tPTeemll3wq0QALAMaYmDKZvWQiYeXKlZSXl0/KvkbbpJ8QSWBjTHzIyMigvr5+1BVdIlBV6uvrycjIGHlhj10BGGNiRkFBATU1NYx3aPh4lZGRQUFBQdjLWwAwxsSM1NRUioqK/C5G3LAmIGOMSVAWAIwxJkFZADDGmAQVU3cCi0gdEN5AG6fKA05EsDixzr6PAfZdDGbfx2Dx8H0sVNVTnqkbUwFgPESkbLhboROVfR8D7LsYzL6PweL5+7AmIGOMSVAWAIwxJkElUgDw5WH0Ucy+jwH2XQxm38dgcft9JEwOwBhjzGCJdAVgjDEmhAUAY4xJUAkRAERkvYi8JiJ7ReR2v8vjFxGZLyLPishuEakSkVv9LlM0EJFkEXlVRH7td1n8JiIzRGSLiOzx/k4u8rtMfhGR27z/k0oR+bmIhD/MZoyI+wAgIsnAvcAVwDLgehFZ5m+pfNML/G9VLQbWAp9P4O8i1K3Abr8LESXuAZ5U1aXAKhL0exGRecAXgVJVXQEkA9f5W6rIi/sAAKwB9qrqPlXtBh4GNvhcJl+o6hFV3e69b8X9c8/zt1T+EpEC4L3Aj/wui99EZBpwKfBjAFXtVtUmf0vlqxRgioikAJnAYZ/LE3GJEADmAdUhn2tI8EoPQEQKgfMB/59L569/Bv4GCPhdkChwNlAH/IfXJPYjEcnyu1B+UNVa4G7gEHAEaFbV3/lbqshLhAAw3LPjErrvq4hkA78EvqSqLX6Xxy8ichVwXFVf8bssUSIFuAC4T1XPB9qBhMyZiUgOrqWgCDgLyBKRj/pbqshLhABQA8wP+VxAHF7KhUtEUnGV/2ZVfdTv8vjsrcD7ReQArmnwnSLyM3+L5KsaoEZVg1eFW3ABIRFdDuxX1TpV7QEeBS72uUwRlwgBYBuwRESKRCQNl8h53Ocy+ULck7R/DOxW1f/rd3n8pqpfU9UCVS3E/V08o6pxd5YXLlU9ClSLyLnepHXALh+L5KdDwFoRyfT+b9YRhwnxuH8kpKr2isgXgKdwmfwHVLXK52L55a3Ax4AKESn3pt2hqk/4WCYTXW4BNnsnS/uAT/pcHl+o6ksisgXYjus99ypxOCSEDQVhjDEJKhGagIwxxgzDAoAxxiQoCwDGGJOgLAAYY0yCsgBgjDEJygKAMcYkKAsAxhiToP4/v70RPHXN0cEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# accuracies\n",
    "plt.plot(r.history['accuracy'], label='acc')\n",
    "plt.plot(r.history['val_accuracy'], label='val_acc')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9750772984673723\n"
     ]
    }
   ],
   "source": [
    "# plot the mean AUC over each label\n",
    "p = model.predict(data)\n",
    "aucs = []\n",
    "for j in range(6):\n",
    "    auc = roc_auc_score(targets[:,j], p[:,j])\n",
    "    aucs.append(auc)\n",
    "print(np.mean(aucs))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}