seismo_performer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "seismo-performer.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "pwV4LjkUy15V",
        "Ua8y-PUSuWyU",
        "a7Or4Dv9zp0p",
        "qPk8M2_4zxo7",
        "XOEVaNXH0D2T",
        "6Mai95nuzJzF",
        "daKR-TJNzcTh",
        "fctdFdmz15tp",
        "CQ55KZ0evqyK",
        "jAhDXNmSDR4w",
        "MdEsA3s1jdpI",
        "THF9GD2jjiaz",
        "CdVUyXd9k5Tr",
        "PQqJxBiFk730",
        "Z0XKelyTlfrv"
      ],
      "machine_shape": "hm",
      "mount_file_id": "1lmhj_E9MOH9INJYLTCQUqJ4l872-AN5U",
      "authorship_tag": "ABX9TyMP/Z5rFjsdtIkfv2H68SsR",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jamm1985/seismo-performer/blob/main/seismo_performer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fjodA5MmpXsz"
      },
      "source": [
        "# Seismo-Performer code examples\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ur-bssxGymiK"
      },
      "source": [
        "# Load the code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dLnjqxJbp3Tj",
        "outputId": "e6900cbb-ae20-43a0-9621-59b058fe6fd0"
      },
      "source": [
        "!pip install einops==0.3.0\n",
        "!pip install obspy==1.2.2\n",
        "!pip install kapre==0.3.5\n",
        "!pip install tensorflow==2.5.0\n",
        "\n",
        "! git clone https://github.com/jamm1985/seismo-performer.git\n",
        "%cd seismo-performer\n",
        "! ls /content/seismo-performer\n",
        "\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pylab as plt\n",
        "\n",
        "import sys\n",
        "import os\n",
        "\n",
        "py_file_location = \"/content/seismo-performer\"\n",
        "sys.path.append(os.path.abspath(py_file_location))\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "import h5py\n",
        "\n",
        "from seismo_performer import load_test_train_data, seismo_performer_with_spec, model_cnn_spec\n",
        "from h5_generator import H5Generator, train_test_split as h5_tts\n",
        "\n",
        "print(tf.__version__)\n",
        "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting einops==0.3.0\n",
            "  Downloading einops-0.3.0-py2.py3-none-any.whl (25 kB)\n",
            "Installing collected packages: einops\n",
            "Successfully installed einops-0.3.0\n",
            "Collecting obspy==1.2.2\n",
            "  Downloading obspy-1.2.2.zip (24.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 24.7 MB 1.5 MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: future>=0.12.4 in /usr/local/lib/python3.7/dist-packages (from obspy==1.2.2) (0.16.0)\n",
            "Requirement already satisfied: scipy>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from obspy==1.2.2) (1.4.1)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from obspy==1.2.2) (4.2.6)\n",
            "Requirement already satisfied: sqlalchemy in /usr/local/lib/python3.7/dist-packages (from obspy==1.2.2) (1.4.20)\n",
            "Requirement already satisfied: matplotlib>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from obspy==1.2.2) (3.2.2)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from obspy==1.2.2) (4.4.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from obspy==1.2.2) (57.2.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from obspy==1.2.2) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.6.1 in /usr/local/lib/python3.7/dist-packages (from obspy==1.2.2) (1.19.5)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=1.1.0->obspy==1.2.2) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=1.1.0->obspy==1.2.2) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=1.1.0->obspy==1.2.2) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=1.1.0->obspy==1.2.2) (0.10.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10->matplotlib>=1.1.0->obspy==1.2.2) (1.15.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->obspy==1.2.2) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->obspy==1.2.2) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->obspy==1.2.2) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->obspy==1.2.2) (1.24.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from sqlalchemy->obspy==1.2.2) (4.6.1)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.7/dist-packages (from sqlalchemy->obspy==1.2.2) (1.1.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->sqlalchemy->obspy==1.2.2) (3.5.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->sqlalchemy->obspy==1.2.2) (3.7.4.3)\n",
            "Building wheels for collected packages: obspy\n",
            "  Building wheel for obspy (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for obspy: filename=obspy-1.2.2-cp37-cp37m-linux_x86_64.whl size=21668675 sha256=8472bef9e9704542fc1241b5c364b5cbdd2ee6ca85517083002f5bce6f562e51\n",
            "  Stored in directory: /root/.cache/pip/wheels/28/7e/ea/0a37d5f5001d096cf97d6527b60300badd2d0074449e89c736\n",
            "Successfully built obspy\n",
            "Installing collected packages: obspy\n",
            "Successfully installed obspy-1.2.2\n",
            "Requirement already satisfied: kapre==0.3.5 in /usr/local/lib/python3.7/dist-packages (0.3.5)\n",
            "Requirement already satisfied: tensorflow>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from kapre==0.3.5) (2.5.0)\n",
            "Requirement already satisfied: librosa>=0.7.2 in /usr/local/lib/python3.7/dist-packages (from kapre==0.3.5) (0.8.1)\n",
            "Requirement already satisfied: numpy>=1.18.5 in /usr/local/lib/python3.7/dist-packages (from kapre==0.3.5) (1.19.5)\n",
            "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (1.4.1)\n",
            "Requirement already satisfied: numba>=0.43.0 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (0.51.2)\n",
            "Requirement already satisfied: scikit-learn!=0.19.0,>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (0.22.2.post1)\n",
            "Requirement already satisfied: decorator>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (4.4.2)\n",
            "Requirement already satisfied: pooch>=1.0 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (1.4.0)\n",
            "Requirement already satisfied: audioread>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (2.1.9)\n",
            "Requirement already satisfied: resampy>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (0.2.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (21.0)\n",
            "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (1.0.1)\n",
            "Requirement already satisfied: soundfile>=0.10.2 in /usr/local/lib/python3.7/dist-packages (from librosa>=0.7.2->kapre==0.3.5) (0.10.3.post1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from numba>=0.43.0->librosa>=0.7.2->kapre==0.3.5) (57.2.0)\n",
            "Requirement already satisfied: llvmlite<0.35,>=0.34.0.dev0 in /usr/local/lib/python3.7/dist-packages (from numba>=0.43.0->librosa>=0.7.2->kapre==0.3.5) (0.34.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->librosa>=0.7.2->kapre==0.3.5) (2.4.7)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from pooch>=1.0->librosa>=0.7.2->kapre==0.3.5) (2.23.0)\n",
            "Requirement already satisfied: appdirs in /usr/local/lib/python3.7/dist-packages (from pooch>=1.0->librosa>=0.7.2->kapre==0.3.5) (1.4.4)\n",
            "Requirement already satisfied: six>=1.3 in /usr/local/lib/python3.7/dist-packages (from resampy>=0.2.2->librosa>=0.7.2->kapre==0.3.5) (1.15.0)\n",
            "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.7/dist-packages (from soundfile>=0.10.2->librosa>=0.7.2->kapre==0.3.5) (1.14.6)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.0->soundfile>=0.10.2->librosa>=0.7.2->kapre==0.3.5) (2.20)\n",
            "Requirement already satisfied: tensorboard~=2.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (2.5.0)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (3.7.4.3)\n",
            "Requirement already satisfied: h5py~=3.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (3.1.0)\n",
            "Requirement already satisfied: grpcio~=1.34.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (1.34.1)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (0.12.0)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (1.12.1)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (0.36.2)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (3.3.0)\n",
            "Requirement already satisfied: gast==0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (0.4.0)\n",
            "Requirement already satisfied: keras-nightly~=2.5.0.dev in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (2.5.0.dev2021032900)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (3.17.3)\n",
            "Requirement already satisfied: tensorflow-estimator<2.6.0,>=2.5.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (2.5.0)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (1.12)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (1.6.3)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (0.2.0)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (1.1.2)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow>=2.0.0->kapre==0.3.5) (1.1.0)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py~=3.1.0->tensorflow>=2.0.0->kapre==0.3.5) (1.5.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (3.3.4)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (1.8.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (0.4.4)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (1.32.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (1.0.1)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (0.6.1)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (4.2.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (4.7.2)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (4.6.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (0.4.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->pooch>=1.0->librosa>=0.7.2->kapre==0.3.5) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->pooch>=1.0->librosa>=0.7.2->kapre==0.3.5) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->pooch>=1.0->librosa>=0.7.2->kapre==0.3.5) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->pooch>=1.0->librosa>=0.7.2->kapre==0.3.5) (2021.5.30)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (3.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.5->tensorflow>=2.0.0->kapre==0.3.5) (3.5.0)\n",
            "Requirement already satisfied: tensorflow==2.5.0 in /usr/local/lib/python3.7/dist-packages (2.5.0)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (0.2.0)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (0.36.2)\n",
            "Requirement already satisfied: h5py~=3.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (3.1.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (3.17.3)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (0.12.0)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (1.6.3)\n",
            "Requirement already satisfied: tensorflow-estimator<2.6.0,>=2.5.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (2.5.0)\n",
            "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (1.15.0)\n",
            "Requirement already satisfied: keras-nightly~=2.5.0.dev in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (2.5.0.dev2021032900)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (1.12)\n",
            "Requirement already satisfied: tensorboard~=2.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (2.5.0)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (1.19.5)\n",
            "Requirement already satisfied: gast==0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (0.4.0)\n",
            "Requirement already satisfied: grpcio~=1.34.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (1.34.1)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (1.12.1)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (3.7.4.3)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (3.3.0)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (1.1.0)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.5.0) (1.1.2)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py~=3.1.0->tensorflow==2.5.0) (1.5.2)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow==2.5.0) (1.8.0)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow==2.5.0) (2.23.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow==2.5.0) (1.0.1)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow==2.5.0) (0.6.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow==2.5.0) (3.3.4)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow==2.5.0) (1.32.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow==2.5.0) (0.4.4)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow==2.5.0) (57.2.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow==2.5.0) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow==2.5.0) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow==2.5.0) (4.2.2)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow==2.5.0) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.5->tensorflow==2.5.0) (4.6.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow==2.5.0) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow==2.5.0) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow==2.5.0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow==2.5.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow==2.5.0) (1.24.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow==2.5.0) (3.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.5->tensorflow==2.5.0) (3.5.0)\n",
            "Cloning into 'seismo-performer'...\n",
            "remote: Enumerating objects: 583, done.\u001b[K\n",
            "remote: Counting objects: 100% (583/583), done.\u001b[K\n",
            "remote: Compressing objects: 100% (374/374), done.\u001b[K\n",
            "remote: Total 583 (delta 356), reused 411 (delta 206), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (583/583), 70.86 MiB | 29.09 MiB/s, done.\n",
            "Resolving deltas: 100% (356/356), done.\n",
            "/content/seismo-performer\n",
            "archive_scan.py  LICENSE      requirements.txt\t   utils\n",
            "fast_attention\t prc_eval.py  seismo_performer.py  WEIGHTS\n",
            "h5_generator.py  README.md    test\n",
            "2.5.0\n",
            "Num GPUs Available:  1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xrZbXg6EzFrJ"
      },
      "source": [
        "# Load datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hBRCEfAuzR7r"
      },
      "source": [
        "## Southern California Ross (filtered/detrended)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hpcZK8FOmHWy"
      },
      "source": [
        "https://scedc.caltech.edu/research-tools/deeplearning.html#phase_detection"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NaFXpk2Cys7z"
      },
      "source": [
        "### copy file to local storage\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9GDbTHeoNne-"
      },
      "source": [
        "# from google drive\n",
        "cp /content/drive/MyDrive/DATA/GFD_fw/scsn_ps_2000_2017_shuf.hdf5 /content/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ifDLFYGPkubQ",
        "outputId": "204bd7d6-aa6a-4f3f-a587-c662ed82956f"
      },
      "source": [
        "# directly from Caltech\n",
        "!wget -O /content/scsn_ps_2000_2017_shuf.hdf5 https://service.scedc.caltech.edu/ftp/ross_etal_2018_bssa/scsn_ps_2000_2017_shuf.hdf5"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-08-13 12:54:43--  https://service.scedc.caltech.edu/ftp/ross_etal_2018_bssa/scsn_ps_2000_2017_shuf.hdf5\n",
            "Resolving service.scedc.caltech.edu (service.scedc.caltech.edu)... 131.215.68.210, 131.215.68.169\n",
            "Connecting to service.scedc.caltech.edu (service.scedc.caltech.edu)|131.215.68.210|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 22918775894 (21G)\n",
            "Saving to: ‘scsn_ps_2000_2017_shuf.hdf5’\n",
            "\n",
            "scsn_ps_2000_2017_s 100%[===================>]  21.34G  13.8MB/s    in 10m 11s \n",
            "\n",
            "2021-08-13 13:04:54 (35.8 MB/s) - ‘scsn_ps_2000_2017_shuf.hdf5’ saved [22918775894/22918775894]\n",
            "\n",
            "/content/: Scheme missing.\n",
            "FINISHED --2021-08-13 13:04:54--\n",
            "Total wall clock time: 10m 11s\n",
            "Downloaded: 1 files, 21G in 10m 11s (35.8 MB/s)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z0pZPik6O2LC",
        "outputId": "2a090aa0-dfdd-4e77-8e7e-d953c8364d6c"
      },
      "source": [
        "ls -l /content/"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 22381636\n",
            "drwx------ 6 root root        4096 Aug 13 12:21 \u001b[0m\u001b[01;34mdrive\u001b[0m/\n",
            "drwxr-xr-x 1 root root        4096 Jul 16 13:20 \u001b[01;34msample_data\u001b[0m/\n",
            "-rw-r--r-- 1 root root 22918775894 Sep  6  2018 scsn_ps_2000_2017_shuf.hdf5\n",
            "drwxr-xr-x 8 root root        4096 Aug 13 13:06 \u001b[01;34mseismo-performer\u001b[0m/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pwV4LjkUy15V"
      },
      "source": [
        "### load example using generator\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yxeoyB-KOLm1"
      },
      "source": [
        " X_train, X_test = h5_tts('/content/scsn_ps_2000_2017_shuf.hdf5',\\\n",
        "                             batch_size = 480,\\\n",
        "                             test_size = 0.2, random_state = 42, shuffle = False)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4OuG7LCgzadB"
      },
      "source": [
        "## Sakhalin and Dagestan filtered/detrended"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_snsaECwmcNw"
      },
      "source": [
        "Please copy datasets to your google drive storage\n",
        "\n",
        "[Sakhalin (HDF5, 127 MB)](https://drive.google.com/file/d/1dH2JF9TQmyB6GpIB_dY1jiWAI5uqp6ED/view?usp=sharing). Total samples 9827: P 3045, S 3737, Noise 3045.\n",
        "\n",
        "[Dagestan (HDF5, 200 MB)](https://drive.google.com/file/d/156w3I9QVnhkCo0u7wjh-c6xekE9f6B3G/view?usp=sharing). Total samples 28111: P 9547, S 9017, Noise 9547.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_jIFfubQwFzB",
        "outputId": "20861656-1bda-484f-cc82-3b10eab22946"
      },
      "source": [
        "# load Sakhalin TEST data M1.0+ fixed channel order\n",
        "_, X_test_sakh, _, y_test_sakh =\\\n",
        "    load_test_train_data('/content/drive/MyDrive/DATA/ML_DATASETS_SAKH_DAG/SAKH-2014_2021_global_norm.h5', 0.99)\n",
        "\n",
        "# load Dagestan data M1.0+ fixed channel order\n",
        "X_train_dag, X_test_dag, y_train_dag, y_test_dag =\\\n",
        "    load_test_train_data('/content/drive/MyDrive/DATA/ML_DATASETS_SAKH_DAG/2021-05-30-DAG_2014_2020_global_norm.h5', 0.99)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [ 0.3094871  0.3803063  0.3102066]%\n",
            "train P, S and noise labels is [ 0.34693878  0.37755102  0.2755102 ]%\n",
            "Total samples 28111\n",
            "P 9547, S 9017, Noise 9547\n",
            "test P, S and noise labels is [ 0.33959756  0.32073302  0.33966942]%\n",
            "train P, S and noise labels is [ 0.34163701  0.32384342  0.33451957]%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-oc-PpT4vdHN"
      },
      "source": [
        "# Train Performer on California Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ua8y-PUSuWyU"
      },
      "source": [
        "### model spec"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dlNHLh9ewC3s",
        "outputId": "9e62dde1-036e-42b4-a8d1-239e8f4bce52"
      },
      "source": [
        "model_with_spec_cal = seismo_performer_with_spec(\n",
        "                                        maxlen=400,\n",
        "                                        nfft=64,\n",
        "                                        hop_length=16,\n",
        "                                        patch_size_1=22,\n",
        "                                        patch_size_2=3,\n",
        "                                        num_channels=3,\n",
        "                                        num_patches=11,\n",
        "                                        d_model=48,\n",
        "                                        num_heads=2,\n",
        "                                        ff_dim_factor=2,\n",
        "                                        layers_depth=2,\n",
        "                                        num_classes=3,\n",
        "                                        drop_out_rate=0.1)\n",
        "\n",
        "LR = 0.0001\n",
        "model_with_spec_cal.compile(\n",
        "    optimizer=keras.optimizers.Adam(learning_rate=LR),\n",
        "    loss=keras.losses.SparseCategoricalCrossentropy(),\n",
        "    metrics=[keras.metrics.SparseCategoricalAccuracy()],)\n",
        "\n",
        "model_with_spec_cal.summary()"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_3 (InputLayer)         [(None, 400, 3)]          0         \n",
            "_________________________________________________________________\n",
            "stft_1 (STFT)                (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude_1 (Magnitude)      (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude_to_decibel_1 (Magn (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "max_abs_scaler_1 (MaxABSScal (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "rearrange3d_1 (Rearrange3d)  (None, 11, 198)           0         \n",
            "_________________________________________________________________\n",
            "dense_11 (Dense)             (None, 11, 48)            9552      \n",
            "_________________________________________________________________\n",
            "cls_token_1 (ClsToken)       (None, 12, 48)            48        \n",
            "_________________________________________________________________\n",
            "pos_embeding2_1 (PosEmbeding (None, 12, 48)            576       \n",
            "_________________________________________________________________\n",
            "performer_block_2 (Performer (None, 12, 48)            18672     \n",
            "_________________________________________________________________\n",
            "performer_block_3 (Performer (None, 12, 48)            18672     \n",
            "_________________________________________________________________\n",
            "lambda_1 (Lambda)            (None, 48)                0         \n",
            "_________________________________________________________________\n",
            "layer_normalization_9 (Layer (None, 48)                96        \n",
            "_________________________________________________________________\n",
            "dropout_11 (Dropout)         (None, 48)                0         \n",
            "_________________________________________________________________\n",
            "dense_16 (Dense)             (None, 96)                4704      \n",
            "_________________________________________________________________\n",
            "dropout_12 (Dropout)         (None, 96)                0         \n",
            "_________________________________________________________________\n",
            "dense_17 (Dense)             (None, 48)                4656      \n",
            "_________________________________________________________________\n",
            "dropout_13 (Dropout)         (None, 48)                0         \n",
            "_________________________________________________________________\n",
            "dense_18 (Dense)             (None, 3)                 147       \n",
            "=================================================================\n",
            "Total params: 57,123\n",
            "Trainable params: 57,123\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MJhJPl3BaPMp"
      },
      "source": [
        "# Load weights example\n",
        "model_with_spec_cal.load_weights('WEIGHTS/w_model_performer_with_spec.hd5')"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CcTEhBqes7BI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2a153dea-6890-4278-daef-2c7b183fcb1e"
      },
      "source": [
        "# evaluate data example with weights\n",
        "\n",
        "print(\"Evaluate transformer based model on SAKHALIN test data\")\n",
        "results_sakh = model_with_spec_cal.evaluate(X_test_sakh, y_test_sakh, batch_size=128)\n",
        "print(\"test loss, test acc:\", results_sakh)\n",
        "\n",
        "print(\"Evaluate transformer based model on DAGESTAN test data\")\n",
        "results_dag = model_with_spec_cal.evaluate(X_test_dag, y_test_dag, batch_size=128)\n",
        "print(\"test loss, test acc:\", results_dag)\n",
        "\n",
        "print(\"Evaluate transformer based model on CALIFORNIA test data\")\n",
        "results_cal = model_with_spec_cal.evaluate(X_test)\n",
        "print(\"test loss, test acc:\", results_cal)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 2s 6ms/step - loss: 0.2640 - sparse_categorical_accuracy: 0.9166\n",
            "test loss, test acc: [0.2639591693878174, 0.9166409969329834]\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 6ms/step - loss: 0.4897 - sparse_categorical_accuracy: 0.8562\n",
            "test loss, test acc: [0.48966699838638306, 0.856162428855896]\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 64s 32ms/step - loss: 0.0444 - sparse_categorical_accuracy: 0.9868\n",
            "test loss, test acc: [0.04436749964952469, 0.9867849946022034]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GnOQd56Y4APM"
      },
      "source": [
        "### Train the model with different random seeds"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTVB2V9lscai",
        "outputId": "be2feb95-0b5b-4641-f591-7db8c7349ff5"
      },
      "source": [
        "%%time\n",
        "\n",
        "interations = 5\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 5, restore_best_weights = True, verbose = 1),\n",
        "    keras.callbacks.ModelCheckpoint('w_performer_epoch_hpa{epoch}.h5', save_weights_only=True),\n",
        "]\n",
        "\n",
        "sparse_categorical_accuracy = []\n",
        "loss = []\n",
        "val_loss = []\n",
        "val_sparse_categorical_accuracy = []\n",
        "\n",
        "cal_top_1 = []\n",
        "sakh_top_1 = []\n",
        "dag_top_1 = []\n",
        "\n",
        "for i in range(interations):\n",
        "  print(\"Load dataset with random state {}\".format(i)) \n",
        "  X_train, X_test = h5_tts('/content/scsn_ps_2000_2017_shuf.hdf5',\\\n",
        "                             batch_size = 480,\\\n",
        "                             test_size = 0.2, random_state = i, shuffle = False)\n",
        "\n",
        "  print(\"compile the model\")\n",
        "  model_with_spec_cal = seismo_performer_with_spec(\n",
        "                                        maxlen=400,\n",
        "                                        nfft=64,\n",
        "                                        hop_length=16,\n",
        "                                        patch_size_1=22,\n",
        "                                        patch_size_2=3,\n",
        "                                        num_channels=3,\n",
        "                                        num_patches=11,\n",
        "                                        d_model=48,\n",
        "                                        num_heads=2,\n",
        "                                        ff_dim_factor=2,\n",
        "                                        layers_depth=2,\n",
        "                                        num_classes=3,\n",
        "                                        drop_out_rate=0.1)\n",
        "\n",
        "\n",
        "\n",
        "  LR = 0.0001\n",
        "  model_with_spec_cal.compile(\n",
        "      optimizer=keras.optimizers.Adam(learning_rate=LR),\n",
        "      loss=keras.losses.SparseCategoricalCrossentropy(),\n",
        "      metrics=[keras.metrics.SparseCategoricalAccuracy()],)\n",
        "\n",
        "  print(\"Fit model on training data step {}\".format(i+1))\n",
        "  history = model_with_spec_cal.fit(\n",
        "    X_train,\n",
        "    #shuffle=False,\n",
        "    epochs=400,\n",
        "    validation_data=(X_test),\n",
        "    callbacks = callbacks\n",
        "    )\n",
        "  \n",
        "  #save convergance curve \n",
        "  sparse_categorical_accuracy.append(history.history['sparse_categorical_accuracy'])\n",
        "  loss.append(history.history['loss'])\n",
        "  val_sparse_categorical_accuracy.append(history.history['val_sparse_categorical_accuracy'])\n",
        "  val_loss.append(history.history['val_loss'])\n",
        "\n",
        "  print(\"Evaluate transformer based model on SAKHALIN test data\")\n",
        "  sakh_top_1.append(model_with_spec_cal.evaluate(X_test_sakh, y_test_sakh, batch_size=128)[1])\n",
        "\n",
        "  print(\"Evaluate transformer based model on DAGESTAN test data\")\n",
        "  dag_top_1.append(model_with_spec_cal.evaluate(X_test_dag, y_test_dag, batch_size=128)[1])\n",
        "  \n",
        "  print(\"Evaluate transformer based model on CALIFORNIA test data\")\n",
        "  cal_top_1.append(model_with_spec_cal.evaluate(X_test)[1])\n",
        "\n",
        "  model_with_spec_cal.save_weights('/content/drive/MyDrive/MODELS/w_model_performer_with_spec_seeds.hd5', save_format='h5')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Load dataset with random state 0\n",
            "compile the model\n",
            "Fit model on training data step 1\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 327s 40ms/step - loss: 0.0936 - sparse_categorical_accuracy: 0.9714 - val_loss: 0.0640 - val_sparse_categorical_accuracy: 0.9806\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 315s 40ms/step - loss: 0.0641 - sparse_categorical_accuracy: 0.9809 - val_loss: 0.0582 - val_sparse_categorical_accuracy: 0.9825\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0588 - sparse_categorical_accuracy: 0.9825 - val_loss: 0.0548 - val_sparse_categorical_accuracy: 0.9836\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 285s 36ms/step - loss: 0.0561 - sparse_categorical_accuracy: 0.9834 - val_loss: 0.0534 - val_sparse_categorical_accuracy: 0.9841\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 282s 35ms/step - loss: 0.0542 - sparse_categorical_accuracy: 0.9839 - val_loss: 0.0534 - val_sparse_categorical_accuracy: 0.9840\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 283s 36ms/step - loss: 0.0528 - sparse_categorical_accuracy: 0.9844 - val_loss: 0.0521 - val_sparse_categorical_accuracy: 0.9843\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 284s 36ms/step - loss: 0.0516 - sparse_categorical_accuracy: 0.9847 - val_loss: 0.0499 - val_sparse_categorical_accuracy: 0.9851\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 285s 36ms/step - loss: 0.0508 - sparse_categorical_accuracy: 0.9850 - val_loss: 0.0497 - val_sparse_categorical_accuracy: 0.9852\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 287s 36ms/step - loss: 0.0500 - sparse_categorical_accuracy: 0.9852 - val_loss: 0.0490 - val_sparse_categorical_accuracy: 0.9853\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 288s 36ms/step - loss: 0.0493 - sparse_categorical_accuracy: 0.9854 - val_loss: 0.0497 - val_sparse_categorical_accuracy: 0.9851\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0487 - sparse_categorical_accuracy: 0.9855 - val_loss: 0.0482 - val_sparse_categorical_accuracy: 0.9856\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 288s 36ms/step - loss: 0.0481 - sparse_categorical_accuracy: 0.9857 - val_loss: 0.0486 - val_sparse_categorical_accuracy: 0.9855\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0477 - sparse_categorical_accuracy: 0.9859 - val_loss: 0.0476 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0472 - sparse_categorical_accuracy: 0.9860 - val_loss: 0.0476 - val_sparse_categorical_accuracy: 0.9857\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 290s 37ms/step - loss: 0.0468 - sparse_categorical_accuracy: 0.9861 - val_loss: 0.0466 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0465 - sparse_categorical_accuracy: 0.9862 - val_loss: 0.0466 - val_sparse_categorical_accuracy: 0.9861\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0461 - sparse_categorical_accuracy: 0.9863 - val_loss: 0.0466 - val_sparse_categorical_accuracy: 0.9861\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0458 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0462 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0454 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0461 - val_sparse_categorical_accuracy: 0.9862\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0452 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0462 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0450 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0454 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0446 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0453 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0444 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0454 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0442 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0450 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0440 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0451 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0449 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0435 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0452 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0433 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 29/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0431 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0444 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 30/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0430 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0451 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 31/400\n",
            "7957/7957 [==============================] - 290s 37ms/step - loss: 0.0428 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0447 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 32/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0426 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0445 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 33/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0425 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 34/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0423 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0442 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 35/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0422 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 36/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0420 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0447 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 37/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0419 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0438 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 38/400\n",
            "7957/7957 [==============================] - 290s 37ms/step - loss: 0.0417 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0437 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 39/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0416 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 40/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0415 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0437 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 41/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0413 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 42/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0412 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0445 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 43/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0411 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 44/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0410 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 45/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0409 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0442 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 46/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0407 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0437 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 47/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0406 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0435 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 48/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0406 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0433 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 49/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0405 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0435 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 50/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0404 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 51/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0403 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 52/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0402 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 53/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0401 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0437 - val_sparse_categorical_accuracy: 0.9872\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00053: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 2s 7ms/step - loss: 0.2903 - sparse_categorical_accuracy: 0.9081\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 2s 7ms/step - loss: 0.4776 - sparse_categorical_accuracy: 0.8548\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 16s 8ms/step - loss: 0.0433 - sparse_categorical_accuracy: 0.9872\n",
            "Load dataset with random state 1\n",
            "compile the model\n",
            "Fit model on training data step 2\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 322s 40ms/step - loss: 0.0935 - sparse_categorical_accuracy: 0.9710 - val_loss: 0.0658 - val_sparse_categorical_accuracy: 0.9802\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 288s 36ms/step - loss: 0.0643 - sparse_categorical_accuracy: 0.9808 - val_loss: 0.0579 - val_sparse_categorical_accuracy: 0.9825\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 287s 36ms/step - loss: 0.0593 - sparse_categorical_accuracy: 0.9824 - val_loss: 0.0556 - val_sparse_categorical_accuracy: 0.9833\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 288s 36ms/step - loss: 0.0565 - sparse_categorical_accuracy: 0.9833 - val_loss: 0.0535 - val_sparse_categorical_accuracy: 0.9839\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 288s 36ms/step - loss: 0.0547 - sparse_categorical_accuracy: 0.9838 - val_loss: 0.0525 - val_sparse_categorical_accuracy: 0.9843\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0533 - sparse_categorical_accuracy: 0.9842 - val_loss: 0.0513 - val_sparse_categorical_accuracy: 0.9846\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0521 - sparse_categorical_accuracy: 0.9846 - val_loss: 0.0509 - val_sparse_categorical_accuracy: 0.9849\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0511 - sparse_categorical_accuracy: 0.9849 - val_loss: 0.0500 - val_sparse_categorical_accuracy: 0.9851\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0504 - sparse_categorical_accuracy: 0.9851 - val_loss: 0.0494 - val_sparse_categorical_accuracy: 0.9854\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0496 - sparse_categorical_accuracy: 0.9854 - val_loss: 0.0490 - val_sparse_categorical_accuracy: 0.9853\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0490 - sparse_categorical_accuracy: 0.9855 - val_loss: 0.0495 - val_sparse_categorical_accuracy: 0.9854\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0484 - sparse_categorical_accuracy: 0.9857 - val_loss: 0.0485 - val_sparse_categorical_accuracy: 0.9854\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0479 - sparse_categorical_accuracy: 0.9858 - val_loss: 0.0475 - val_sparse_categorical_accuracy: 0.9858\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0474 - sparse_categorical_accuracy: 0.9860 - val_loss: 0.0478 - val_sparse_categorical_accuracy: 0.9857\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0470 - sparse_categorical_accuracy: 0.9861 - val_loss: 0.0472 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0467 - sparse_categorical_accuracy: 0.9862 - val_loss: 0.0470 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0463 - sparse_categorical_accuracy: 0.9863 - val_loss: 0.0481 - val_sparse_categorical_accuracy: 0.9855\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0459 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0471 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0457 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0459 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0454 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0459 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0452 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0462 - val_sparse_categorical_accuracy: 0.9862\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0449 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0457 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0445 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0452 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0443 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0462 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0442 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0458 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0439 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0451 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0453 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0435 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0456 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 29/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0433 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0450 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 30/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0431 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0451 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 31/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0429 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0455 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 32/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0428 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 33/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0427 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0445 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 34/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0425 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0450 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 35/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0423 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 36/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0422 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0458 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 37/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0420 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0454 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 38/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0419 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 39/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0418 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0441 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 40/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0416 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0441 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 41/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0415 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 42/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0414 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0450 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 43/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0413 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 44/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0412 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0441 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 45/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0410 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0441 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 46/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0409 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 47/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0408 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 48/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0407 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 49/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0407 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0434 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 50/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0405 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0437 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 51/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0404 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0441 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 52/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0403 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 53/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0402 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0445 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 54/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0401 - sparse_categorical_accuracy: 0.9883 - val_loss: 0.0438 - val_sparse_categorical_accuracy: 0.9871\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00054: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 6ms/step - loss: 0.3229 - sparse_categorical_accuracy: 0.9050\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 6ms/step - loss: 0.5409 - sparse_categorical_accuracy: 0.8448\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 14s 7ms/step - loss: 0.0434 - sparse_categorical_accuracy: 0.9871\n",
            "Load dataset with random state 2\n",
            "compile the model\n",
            "Fit model on training data step 3\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 322s 40ms/step - loss: 0.0931 - sparse_categorical_accuracy: 0.9709 - val_loss: 0.0654 - val_sparse_categorical_accuracy: 0.9802\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0641 - sparse_categorical_accuracy: 0.9809 - val_loss: 0.0584 - val_sparse_categorical_accuracy: 0.9825\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0589 - sparse_categorical_accuracy: 0.9825 - val_loss: 0.0553 - val_sparse_categorical_accuracy: 0.9833\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0562 - sparse_categorical_accuracy: 0.9834 - val_loss: 0.0534 - val_sparse_categorical_accuracy: 0.9839\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0542 - sparse_categorical_accuracy: 0.9839 - val_loss: 0.0526 - val_sparse_categorical_accuracy: 0.9842\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0529 - sparse_categorical_accuracy: 0.9843 - val_loss: 0.0507 - val_sparse_categorical_accuracy: 0.9849\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0518 - sparse_categorical_accuracy: 0.9847 - val_loss: 0.0503 - val_sparse_categorical_accuracy: 0.9849\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0508 - sparse_categorical_accuracy: 0.9850 - val_loss: 0.0497 - val_sparse_categorical_accuracy: 0.9852\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0500 - sparse_categorical_accuracy: 0.9852 - val_loss: 0.0493 - val_sparse_categorical_accuracy: 0.9853\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0492 - sparse_categorical_accuracy: 0.9854 - val_loss: 0.0483 - val_sparse_categorical_accuracy: 0.9855\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0487 - sparse_categorical_accuracy: 0.9856 - val_loss: 0.0483 - val_sparse_categorical_accuracy: 0.9855\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0481 - sparse_categorical_accuracy: 0.9858 - val_loss: 0.0478 - val_sparse_categorical_accuracy: 0.9856\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0476 - sparse_categorical_accuracy: 0.9859 - val_loss: 0.0476 - val_sparse_categorical_accuracy: 0.9857\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0472 - sparse_categorical_accuracy: 0.9861 - val_loss: 0.0469 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0468 - sparse_categorical_accuracy: 0.9862 - val_loss: 0.0478 - val_sparse_categorical_accuracy: 0.9857\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0464 - sparse_categorical_accuracy: 0.9863 - val_loss: 0.0467 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0460 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0463 - val_sparse_categorical_accuracy: 0.9862\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0457 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0466 - val_sparse_categorical_accuracy: 0.9861\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0454 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0460 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0451 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0458 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0448 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0457 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0446 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0452 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0444 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0454 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0441 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0453 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0439 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0454 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0450 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0435 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0432 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0447 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 29/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0430 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0451 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 30/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0429 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 31/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0427 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0452 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 32/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0426 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 33/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0424 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0444 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 34/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0422 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 35/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0420 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 36/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0419 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 37/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0418 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9868\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00037: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 7ms/step - loss: 0.3005 - sparse_categorical_accuracy: 0.9088\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 7ms/step - loss: 0.5229 - sparse_categorical_accuracy: 0.8455\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 15s 7ms/step - loss: 0.0443 - sparse_categorical_accuracy: 0.9869\n",
            "Load dataset with random state 3\n",
            "compile the model\n",
            "Fit model on training data step 4\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 324s 40ms/step - loss: 0.0930 - sparse_categorical_accuracy: 0.9712 - val_loss: 0.0649 - val_sparse_categorical_accuracy: 0.9803\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0647 - sparse_categorical_accuracy: 0.9807 - val_loss: 0.0585 - val_sparse_categorical_accuracy: 0.9824\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 287s 36ms/step - loss: 0.0594 - sparse_categorical_accuracy: 0.9823 - val_loss: 0.0558 - val_sparse_categorical_accuracy: 0.9831\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0566 - sparse_categorical_accuracy: 0.9832 - val_loss: 0.0534 - val_sparse_categorical_accuracy: 0.9840\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0547 - sparse_categorical_accuracy: 0.9838 - val_loss: 0.0522 - val_sparse_categorical_accuracy: 0.9843\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0532 - sparse_categorical_accuracy: 0.9843 - val_loss: 0.0515 - val_sparse_categorical_accuracy: 0.9845\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0521 - sparse_categorical_accuracy: 0.9846 - val_loss: 0.0505 - val_sparse_categorical_accuracy: 0.9849\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0510 - sparse_categorical_accuracy: 0.9850 - val_loss: 0.0495 - val_sparse_categorical_accuracy: 0.9852\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0502 - sparse_categorical_accuracy: 0.9851 - val_loss: 0.0493 - val_sparse_categorical_accuracy: 0.9853\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0496 - sparse_categorical_accuracy: 0.9854 - val_loss: 0.0482 - val_sparse_categorical_accuracy: 0.9855\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0490 - sparse_categorical_accuracy: 0.9855 - val_loss: 0.0479 - val_sparse_categorical_accuracy: 0.9856\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0484 - sparse_categorical_accuracy: 0.9857 - val_loss: 0.0474 - val_sparse_categorical_accuracy: 0.9858\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0479 - sparse_categorical_accuracy: 0.9859 - val_loss: 0.0479 - val_sparse_categorical_accuracy: 0.9855\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0474 - sparse_categorical_accuracy: 0.9860 - val_loss: 0.0473 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0469 - sparse_categorical_accuracy: 0.9861 - val_loss: 0.0472 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0466 - sparse_categorical_accuracy: 0.9862 - val_loss: 0.0465 - val_sparse_categorical_accuracy: 0.9861\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0463 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0466 - val_sparse_categorical_accuracy: 0.9862\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0459 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0458 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0455 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0462 - val_sparse_categorical_accuracy: 0.9862\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0452 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0460 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0449 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0457 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0447 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0455 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0444 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0454 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0443 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0451 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0440 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0436 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0454 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0435 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0451 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0433 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 29/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0432 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 30/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0430 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0447 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 31/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0428 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 32/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0426 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0444 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 33/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0425 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 34/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0423 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 35/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0422 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0442 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 36/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0421 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 37/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0418 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 38/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0418 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0444 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 39/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0417 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0441 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 40/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0415 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0442 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 41/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0414 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0445 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 42/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0413 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0442 - val_sparse_categorical_accuracy: 0.9869\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00042: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 6ms/step - loss: 0.2919 - sparse_categorical_accuracy: 0.9082\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 6ms/step - loss: 0.5352 - sparse_categorical_accuracy: 0.8434\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 14s 7ms/step - loss: 0.0439 - sparse_categorical_accuracy: 0.9870\n",
            "Load dataset with random state 4\n",
            "compile the model\n",
            "Fit model on training data step 5\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 324s 40ms/step - loss: 0.0918 - sparse_categorical_accuracy: 0.9719 - val_loss: 0.0657 - val_sparse_categorical_accuracy: 0.9801\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 288s 36ms/step - loss: 0.0639 - sparse_categorical_accuracy: 0.9810 - val_loss: 0.0580 - val_sparse_categorical_accuracy: 0.9826\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 288s 36ms/step - loss: 0.0589 - sparse_categorical_accuracy: 0.9825 - val_loss: 0.0549 - val_sparse_categorical_accuracy: 0.9836\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0561 - sparse_categorical_accuracy: 0.9834 - val_loss: 0.0536 - val_sparse_categorical_accuracy: 0.9838\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0543 - sparse_categorical_accuracy: 0.9839 - val_loss: 0.0516 - val_sparse_categorical_accuracy: 0.9845\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0528 - sparse_categorical_accuracy: 0.9844 - val_loss: 0.0511 - val_sparse_categorical_accuracy: 0.9847\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0517 - sparse_categorical_accuracy: 0.9847 - val_loss: 0.0509 - val_sparse_categorical_accuracy: 0.9846\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0509 - sparse_categorical_accuracy: 0.9849 - val_loss: 0.0495 - val_sparse_categorical_accuracy: 0.9851\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0500 - sparse_categorical_accuracy: 0.9852 - val_loss: 0.0490 - val_sparse_categorical_accuracy: 0.9852\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0493 - sparse_categorical_accuracy: 0.9854 - val_loss: 0.0518 - val_sparse_categorical_accuracy: 0.9844\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0487 - sparse_categorical_accuracy: 0.9856 - val_loss: 0.0483 - val_sparse_categorical_accuracy: 0.9856\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0481 - sparse_categorical_accuracy: 0.9857 - val_loss: 0.0483 - val_sparse_categorical_accuracy: 0.9855\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0476 - sparse_categorical_accuracy: 0.9859 - val_loss: 0.0471 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0472 - sparse_categorical_accuracy: 0.9860 - val_loss: 0.0469 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0467 - sparse_categorical_accuracy: 0.9862 - val_loss: 0.0473 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0464 - sparse_categorical_accuracy: 0.9863 - val_loss: 0.0469 - val_sparse_categorical_accuracy: 0.9861\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0461 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0464 - val_sparse_categorical_accuracy: 0.9862\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0457 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0467 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0454 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0458 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0451 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0456 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0448 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0460 - val_sparse_categorical_accuracy: 0.9864\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0446 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0456 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0443 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0461 - val_sparse_categorical_accuracy: 0.9862\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0441 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0451 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0439 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0455 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0450 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0434 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0432 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 29/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0430 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 30/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0428 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 31/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0427 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 32/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0425 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0445 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 33/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0424 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0449 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 34/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0422 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0442 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 35/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0420 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0444 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 36/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0419 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 37/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0417 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 38/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0416 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0441 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 39/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0415 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 40/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0414 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 41/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0412 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0447 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 42/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0411 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 43/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0410 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0438 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 44/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0409 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0437 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 45/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0407 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0434 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 46/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0407 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0444 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 47/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0405 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0433 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 48/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0404 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 49/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0403 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0435 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 50/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0403 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0435 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 51/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0401 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 52/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0400 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0430 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 53/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0399 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0432 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 54/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0398 - sparse_categorical_accuracy: 0.9883 - val_loss: 0.0431 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 55/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0397 - sparse_categorical_accuracy: 0.9883 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 56/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0396 - sparse_categorical_accuracy: 0.9884 - val_loss: 0.0431 - val_sparse_categorical_accuracy: 0.9874\n",
            "Epoch 57/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0395 - sparse_categorical_accuracy: 0.9884 - val_loss: 0.0431 - val_sparse_categorical_accuracy: 0.9874\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00057: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 2s 7ms/step - loss: 0.3176 - sparse_categorical_accuracy: 0.9042\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 7ms/step - loss: 0.5792 - sparse_categorical_accuracy: 0.8342\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 15s 7ms/step - loss: 0.0430 - sparse_categorical_accuracy: 0.9873\n",
            "CPU times: user 13h 31min 33s, sys: 2h 36min 13s, total: 16h 7min 46s\n",
            "Wall time: 19h 49min 15s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bV2QY4q_4rv-"
      },
      "source": [
        "### Print collected values and scores"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u3fHtpZ_gY2M",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "09982a22-c0ee-4a21-e295-9bfa3bb61d79"
      },
      "source": [
        "print(val_sparse_categorical_accuracy)\n",
        "print(val_loss)\n",
        "print(sakh_top_1)\n",
        "print(dag_top_1)\n",
        "print(cal_top_1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.9806427955627441, 0.9824507832527161, 0.9835594892501831, 0.9840986132621765, 0.9840022921562195, 0.9842545986175537, 0.9850701689720154, 0.9851507544517517, 0.9853454828262329, 0.9850764274597168, 0.9856459498405457, 0.9855046272277832, 0.9858605265617371, 0.985746443271637, 0.9860228300094604, 0.9860950708389282, 0.9861055016517639, 0.9862730503082275, 0.9862426519393921, 0.9862803816795349, 0.9864227175712585, 0.9865043759346008, 0.9864352941513062, 0.9866457581520081, 0.9865316152572632, 0.986757755279541, 0.9865201115608215, 0.9866101145744324, 0.986815333366394, 0.9865630269050598, 0.9866321086883545, 0.98683100938797, 0.9868917465209961, 0.9869419932365417, 0.9868550896644592, 0.9869692325592041, 0.9870508909225464, 0.9869995713233948, 0.9869409799575806, 0.987063467502594, 0.9869419932365417, 0.9867786765098572, 0.9869912266731262, 0.9871880412101746, 0.9868907332420349, 0.9870383143424988, 0.9871105551719666, 0.9871880412101746, 0.9872058033943176, 0.9870728850364685, 0.9870665669441223, 0.9870194792747498, 0.9871587157249451], [0.9802135825157166, 0.9824885129928589, 0.9833333492279053, 0.9839394688606262, 0.9842577576637268, 0.9846210479736328, 0.9848617911338806, 0.9850858449935913, 0.9853549003601074, 0.9853349924087524, 0.9853789806365967, 0.9853821396827698, 0.98580402135849, 0.9857223629951477, 0.9859568476676941, 0.9859369993209839, 0.9855402112007141, 0.985996663570404, 0.9863902926445007, 0.9864059686660767, 0.9862070679664612, 0.9864740371704102, 0.9865357875823975, 0.9863379597663879, 0.9864447116851807, 0.9866488575935364, 0.9865598678588867, 0.9864970445632935, 0.9867441654205322, 0.9866446852684021, 0.9866164326667786, 0.9866740107536316, 0.9869032502174377, 0.9867127537727356, 0.9867954254150391, 0.9863379597663879, 0.9866384267807007, 0.9869346618652344, 0.9869964122772217, 0.986931562423706, 0.986947238445282, 0.9867430925369263, 0.9869985580444336, 0.9870372414588928, 0.9870875477790833, 0.9871188998222351, 0.9870173931121826, 0.9871628880500793, 0.9871137142181396, 0.987136721611023, 0.9869713187217712, 0.9870582222938538, 0.9868383407592773, 0.9871220588684082], [0.9802030920982361, 0.9824832677841187, 0.9833019375801086, 0.9839133024215698, 0.9842043519020081, 0.9848858714103699, 0.9848565459251404, 0.9851664304733276, 0.9852585792541504, 0.9855014681816101, 0.9855056405067444, 0.9856019616127014, 0.9856574535369873, 0.9859820008277893, 0.9857370257377625, 0.9860102534294128, 0.986243724822998, 0.9861338138580322, 0.9862719774246216, 0.9863882064819336, 0.9863274693489075, 0.9865022897720337, 0.9865086078643799, 0.9864112138748169, 0.9864949584007263, 0.9866645932197571, 0.9866865873336792, 0.9867075085639954, 0.9866771101951599, 0.9868226647377014, 0.9865787029266357, 0.9869095683097839, 0.9868215918540955, 0.9867221713066101, 0.9868080019950867, 0.9868362545967102, 0.9867797493934631], [0.9802847504615784, 0.9823911190032959, 0.9831344485282898, 0.9839719533920288, 0.9843299984931946, 0.9845194816589355, 0.9848555326461792, 0.9852062463760376, 0.9852690696716309, 0.9855297207832336, 0.9855904579162598, 0.9857987761497498, 0.9855338931083679, 0.9858856797218323, 0.9859976768493652, 0.9860950708389282, 0.9862039089202881, 0.9863379597663879, 0.986201822757721, 0.9863128066062927, 0.9863850474357605, 0.9864729642868042, 0.9864970445632935, 0.9865378737449646, 0.986660361289978, 0.9865117073059082, 0.986792266368866, 0.9866719245910645, 0.9867692589759827, 0.9868258237838745, 0.9869001507759094, 0.9867944121360779, 0.9867535829544067, 0.9867828488349915, 0.9870215654373169, 0.987023651599884, 0.9870445728302002, 0.986928403377533, 0.9870204925537109, 0.98677659034729, 0.9868059158325195, 0.9869440793991089], [0.9800523519515991, 0.9825544357299805, 0.98355633020401, 0.9838442206382751, 0.9844671487808228, 0.9846827983856201, 0.9845707416534424, 0.9850555062294006, 0.9851831793785095, 0.984375, 0.985576868057251, 0.9854648113250732, 0.9860008358955383, 0.9859788417816162, 0.9858898520469666, 0.9861107468605042, 0.9861976504325867, 0.9860343337059021, 0.9864217042922974, 0.9863588809967041, 0.9863609671592712, 0.9864939451217651, 0.9862008094787598, 0.9866823554039001, 0.9865735173225403, 0.9867284297943115, 0.986737847328186, 0.9866876006126404, 0.9866740107536316, 0.9866499304771423, 0.98683100938797, 0.9867944121360779, 0.9867755174636841, 0.9869242310523987, 0.9867483377456665, 0.9866216778755188, 0.9869451522827148, 0.9869577288627625, 0.987062394618988, 0.9870728850364685, 0.9867410063743591, 0.9871764779090881, 0.9870121479034424, 0.9871021509170532, 0.9871576428413391, 0.987003743648529, 0.9872298836708069, 0.9870550632476807, 0.9871848821640015, 0.9871126413345337, 0.9871608018875122, 0.9872676134109497, 0.9872267842292786, 0.9873471260070801, 0.9872078895568848, 0.9873743653297424, 0.9873649477958679]]\n",
            "[[0.06396667659282684, 0.05815700441598892, 0.05479487404227257, 0.05335971340537071, 0.053352661430835724, 0.05213392153382301, 0.04985028877854347, 0.04965035617351532, 0.04903937131166458, 0.049663007259368896, 0.048165421932935715, 0.04855287820100784, 0.047566238790750504, 0.04757801443338394, 0.046564217656850815, 0.046557288616895676, 0.04657401889562607, 0.04621807485818863, 0.046140335500240326, 0.04616047814488411, 0.04541488736867905, 0.04528293386101723, 0.045382168143987656, 0.0449516735970974, 0.04508815333247185, 0.044902559369802475, 0.0451953262090683, 0.04482225701212883, 0.04438452050089836, 0.04507700353860855, 0.044699527323246, 0.044497549533843994, 0.04433584585785866, 0.04419822245836258, 0.0440259724855423, 0.04466596618294716, 0.04377202317118645, 0.04371597245335579, 0.04403962194919586, 0.0437023900449276, 0.04396567493677139, 0.04450284317135811, 0.04364275559782982, 0.04392530396580696, 0.04420221969485283, 0.043695118278265, 0.04351213574409485, 0.043255072087049484, 0.04352027177810669, 0.04458068311214447, 0.044642988592386246, 0.04397840425372124, 0.04366816580295563], [0.06576674431562424, 0.05794041231274605, 0.055612269788980484, 0.05347057431936264, 0.05247287452220917, 0.051263414323329926, 0.05090271681547165, 0.04999389871954918, 0.04941082000732422, 0.04900286719202995, 0.04953727126121521, 0.048505958169698715, 0.04748455062508583, 0.047803085297346115, 0.047224536538124084, 0.04696649685502052, 0.04808015748858452, 0.047074686735868454, 0.045896973460912704, 0.04585972800850868, 0.046226564794778824, 0.04565635323524475, 0.04516978934407234, 0.04619699344038963, 0.04581659287214279, 0.045055270195007324, 0.04530753567814827, 0.045563504099845886, 0.044975098222494125, 0.045063234865665436, 0.04550376534461975, 0.044644638895988464, 0.04445256292819977, 0.04497021809220314, 0.04455489292740822, 0.04578967019915581, 0.04536230489611626, 0.04431118071079254, 0.04409466311335564, 0.04413653910160065, 0.04402150958776474, 0.04499228671193123, 0.04394359886646271, 0.04412642493844032, 0.04411178082227707, 0.043965574353933334, 0.04404739290475845, 0.0436030775308609, 0.04340961575508118, 0.04371261969208717, 0.04408785328269005, 0.04403162747621536, 0.04451972618699074, 0.04382000491023064], [0.06537371873855591, 0.05843095853924751, 0.055312804877758026, 0.053429268300533295, 0.05255693569779396, 0.050741977989673615, 0.050320059061050415, 0.0496881939470768, 0.049297068268060684, 0.04834641143679619, 0.04832182452082634, 0.047759391367435455, 0.04758383333683014, 0.04693392664194107, 0.04776305705308914, 0.04674242064356804, 0.046284034848213196, 0.04663641378283501, 0.04596365615725517, 0.045805297791957855, 0.04569981247186661, 0.04524509236216545, 0.0453822948038578, 0.04531002417206764, 0.0453992635011673, 0.04498574510216713, 0.04464803636074066, 0.044684555381536484, 0.045136287808418274, 0.044629622250795364, 0.04521367698907852, 0.04426690191030502, 0.04437746852636337, 0.04480358585715294, 0.04478335753083229, 0.04430767893791199, 0.04478234797716141], [0.06486291438341141, 0.058472562581300735, 0.05583641305565834, 0.05342838540673256, 0.05215708911418915, 0.05151066184043884, 0.05048425495624542, 0.04953126609325409, 0.04934833198785782, 0.04816111549735069, 0.047883063554763794, 0.047414083033800125, 0.04790177941322327, 0.047342732548713684, 0.047181136906147, 0.046495404094457626, 0.04662409797310829, 0.04580464959144592, 0.04617420956492424, 0.04598639905452728, 0.04574471712112427, 0.04546334221959114, 0.04542103409767151, 0.04511869326233864, 0.044837966561317444, 0.04535142332315445, 0.04508489742875099, 0.04481176659464836, 0.0447721853852272, 0.044670701026916504, 0.044312793761491776, 0.044411733746528625, 0.04464508593082428, 0.04429244622588158, 0.044177304953336716, 0.04392477497458458, 0.0438726507127285, 0.04441014304757118, 0.04408904165029526, 0.04424780234694481, 0.04445405304431915, 0.044164784252643585], [0.06569068878889084, 0.05799981206655502, 0.0548972561955452, 0.05360161513090134, 0.051589641720056534, 0.05107085779309273, 0.0508609302341938, 0.04951252415776253, 0.049025047570466995, 0.05176049470901489, 0.048257965594530106, 0.048291515558958054, 0.04710874706506729, 0.04689907282590866, 0.047250229865312576, 0.04691370204091072, 0.04639517515897751, 0.046682436019182205, 0.04581396281719208, 0.04561489447951317, 0.046023931354284286, 0.045636918395757675, 0.04605262354016304, 0.04514268785715103, 0.04553605243563652, 0.04496549442410469, 0.04477614536881447, 0.04481588676571846, 0.044838450849056244, 0.04459770768880844, 0.044617313891649246, 0.04447250813245773, 0.04489856958389282, 0.04418729245662689, 0.04437115043401718, 0.04480753839015961, 0.04392527416348457, 0.04408232122659683, 0.04386730119585991, 0.043972134590148926, 0.04468899220228195, 0.04362178221344948, 0.04377743601799011, 0.043707605451345444, 0.043432462960481644, 0.044366005808115005, 0.04334932193160057, 0.044303473085165024, 0.04350731521844864, 0.04349399730563164, 0.04361579939723015, 0.0429559163749218, 0.04315243288874626, 0.04313479736447334, 0.043887898325920105, 0.043124739080667496, 0.043138958513736725]]\n",
            "[0.9081097841262817, 0.9050261974334717, 0.9088292717933655, 0.9082125425338745, 0.9042039513587952]\n",
            "[0.8547610640525818, 0.8448077440261841, 0.8454545736312866, 0.8434423208236694, 0.8341717720031738]\n",
            "[0.9871880412101746, 0.9871137142181396, 0.9869095683097839, 0.9870445728302002, 0.9872676134109497]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IECMzir-40wt",
        "outputId": "1f8343cf-6f1b-408e-c593-e9134cc5e86c"
      },
      "source": [
        "print(\"Mean SAKHALIN test TOP-1 accuracy {}, std. dev {}\".format(np.mean(sakh_top_1), np.std(sakh_top_1)))\n",
        "print(\"Mean DAGESTAN test TOP-1 accuracy {}, std. dev {}\".format(np.mean(dag_top_1), np.std(dag_top_1)))\n",
        "print(\"Mean CALIFIRNIA test TOP-1 accuracy {}, std. dev {}\".format(np.mean(cal_top_1), np.std(cal_top_1)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mean SAKHALIN test TOP-1 accuracy 0.9068763494491577, std. dev 0.0018807154372420872\n",
            "Mean DAGESTAN test TOP-1 accuracy 0.8445274949073791, std. dev 0.006543432610475853\n",
            "Mean CALIFIRNIA test TOP-1 accuracy 0.9871047019958497, std. dev 0.00012268614099733774\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J3OWErwuvPJS"
      },
      "source": [
        "### Plot convergence curves for latest iteration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q9XubSPbsi4a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 590
        },
        "outputId": "4f782993-7a21-4dfb-aa97-23c59649457f"
      },
      "source": [
        "# list all data in history\n",
        "print(history.history.keys())\n",
        "# summarize history for accuracy\n",
        "plt.plot(history.history['sparse_categorical_accuracy'])\n",
        "plt.plot(history.history['val_sparse_categorical_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper left')\n",
        "plt.show()\n",
        "# summarize history for loss\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dict_keys(['loss', 'sparse_categorical_accuracy', 'val_loss', 'val_sparse_categorical_accuracy'])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEWCAYAAACufwpNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXyU5bn4/8+VZLKHhCysYROQRUQQRFwQBK3gvtXi1tpzLG21p9rWfqun1bb+jtXTY21rtbXW2mpV1GJVVFwQwaVu7AiyRQSSACEEkpBtkplcvz/uJ2ESAgzJDNmu9+s1r5l51vvG+Fxz76KqGGOMMZEQ094JMMYY03VYUDHGGBMxFlSMMcZEjAUVY4wxEWNBxRhjTMRYUDHGGBMxFlSMaSUR+buI/E+Yx24VkXOinSZj2psFFWOMMRFjQcWYbk5E4to7DabrsKBiujSv2unHIrJGRCpF5K8i0ltEXheR/SLytoj0DDn+YhFZJyKlIrJEREaF7BsvIiu8854DEpvd60IRWeWd+6GIjA0zjReIyEoRKReRfBH5RbP9Z3rXK/X23+BtTxKR34jINhEpE5EPvG3TRKSghX+Hc7zPvxCReSLylIiUAzeIyCQR+ci7x04ReUhE4kPOP0FEForIXhEpEpH/FpE+IlIlIlkhx50sIsUi4gsn76brsaBiuoMrgHOB44GLgNeB/wZycP8PfB9ARI4H5gK3evsWAK+ISLz3gH0J+AeQCfzTuy7eueOBx4FvA1nAn4H5IpIQRvoqga8DGcAFwHdF5FLvuoO89P7BS9M4YJV33v3ABOB0L03/D6gP89/kEmCed8+ngSDwAyAbOA2YAdzkpSENeBt4A+gHDAMWqeouYAlwVch1rweeVdW6MNNhuhgLKqY7+IOqFqlqIfA+8ImqrlTVGuBFYLx33NeA11R1ofdQvB9Iwj20JwM+4HeqWqeq84ClIfeYA/xZVT9R1aCqPgH4vfMOS1WXqOpnqlqvqmtwgW2qt/sa4G1Vnevdt0RVV4lIDPAfwC2qWujd80NV9Yf5b/KRqr7k3bNaVZer6seqGlDVrbig2JCGC4FdqvobVa1R1f2q+om37wngOgARiQWuxgVe001ZUDHdQVHI5+oWvqd6n/sB2xp2qGo9kA/09/YVatMZWLeFfB4E/MirPioVkVJggHfeYYnIqSKy2Ks2KgO+gysx4F3jixZOy8ZVv7W0Lxz5zdJwvIi8KiK7vCqxX4WRBoCXgdEiMgRXGixT1U9bmSbTBVhQMeaAHbjgAICICO6BWgjsBPp72xoMDPmcD9yjqhkhr2RVnRvGfZ8B5gMDVDUdeARouE8+MLSFc/YANYfYVwkkh+QjFld1Fqr59OR/AjYAw1W1B656MDQNx7WUcK+09zyutHI9Vkrp9iyoGHPA88AFIjLDa2j+Ea4K60PgIyAAfF9EfCJyOTAp5Ny/AN/xSh0iIileA3xaGPdNA/aqao2ITMJVeTV4GjhHRK4SkTgRyRKRcV4p6nHgARHpJyKxInKa14azCUj07u8DfgYcqW0nDSgHKkRkJPDdkH2vAn1F5FYRSRCRNBE5NWT/k8ANwMVYUOn2LKgY41HVjbhf3H/AlQQuAi5S1VpVrQUuxz089+LaX/4Vcu4y4FvAQ8A+IM87Nhw3AXeLyH7gLlxwa7juduB8XIDbi2ukP8nbfRvwGa5tZy/wv0CMqpZ513wMV8qqBJr0BmvBbbhgth8XIJ8LScN+XNXWRcAuYDNwdsj+f+M6CKxQ1dAqQdMNiS3SZYxpKxF5B3hGVR9r77SY9mVBxRjTJiJyCrAQ1ya0v73TY9qXVX8ZY1pNRJ7AjWG51QKKASupGGOMiSArqRhjjImYbj2RXHZ2tg4ePLi9k2GMMZ3K8uXL96hq87FPQDcPKoMHD2bZsmXtnQxjjOlUROSQXcet+ssYY0zEWFAxxhgTMRZUjDHGREy3blNpSV1dHQUFBdTU1LR3UqIuMTGR3NxcfD5bT8kYExkWVJopKCggLS2NwYMH03RC2q5FVSkpKaGgoIAhQ4a0d3KMMV2EVX81U1NTQ1ZWVpcOKAAiQlZWVrcokRljjh0LKi3o6gGlQXfJpzHm2LHqL2OM6eTq65UteyopKq+hwh+g0ntV+IP4A0GSfLEkJ8SR7IslJSGWpPg4BmclMygrJeJpsaDSwZSWlvLMM89w0003HdV5559/Ps888wwZGRlRSpkxpj2oKsF6JVB/4L3CH2BtYRmr80tZXVDKmvwy9vsDR3Xd70wdyu2zRkY8vRZUOpjS0lL++Mc/HhRUAoEAcXGH/s+1YMGCaCfNGNMKFf4A+XurKNhX3fheUunHX1ePPxDEH6j3XkFq6uqpqXPv/kAQf109tcH6Q147LkYY2TeNS8b346TcDAZkJpOaEEdqQhwp3nt8XAw1dUEqawNU1wap9AeprguQk5oYlfxaUOlgbr/9dr744gvGjRuHz+cjMTGRnj17smHDBjZt2sSll15Kfn4+NTU13HLLLcyZMwc4MOVMRUUFs2bN4swzz+TDDz+kf//+vPzyyyQlJbVzzozpvMqq6ti+t4r8fVUU7KvCX1dPbKwQFyPExsQQKxCoV4or/BSX+9m9309ReQ279/spq65rcq0kXyy9eiSQGBdLgi+GhLgYknyxpCf5SPTFhGyPJdEXS3ysEBcbQ2xMw/2ERF8so/qmcUK/dBJ9sUdMf4oXZI4FCyqH8ctX1vH5jvKIXnN0vx78/KITDrn/vvvuY+3ataxatYolS5ZwwQUXsHbt2sZuv48//jiZmZlUV1dzyimncMUVV5CVldXkGps3b2bu3Ln85S9/4aqrruKFF17guuuui2g+jOlM6oL1bNy1n88Ky9hcVEF8XAw9kuLokeijR5KPHolx1AW1MRDs9t53ldWQv6+K/TXhVS3Fx8aQk5ZArx4JHJeTwuTjsuiXkcSAzCRyeyYzoGcSmSnxXbqTjAWVDm7SpElNxpE8+OCDvPjiiwDk5+ezefPmg4LKkCFDGDduHAATJkxg69atxyy9xrQHVaWsuo7i/X5XWtjvXltLKvmssJz1O8upDbhqpCRfLIH6euqCLa8lJQJZKQn0Skugd48EJg7uyYCeyQcCQ2YyKfGxTdo46usVEUhP8nXpgBEOCyqHcbgSxbGSknKgd8aSJUt4++23+eijj0hOTmbatGktjjNJSEho/BwbG0t1dfUxSasxkaCq7PcHKN7vp6ishp1lNewqr2FHaTW7ymrYU+Gnui7oXrWuDaK6Lkiw/uAgkZoQx5j+Pbjh9MGM6Z/O2P7pDMxMRgT8gXrKq+sorwlQVl1HXIzQu0ciWanx+GKPPNoi7si1Tt2SBZUOJi0tjf37W16VtaysjJ49e5KcnMyGDRv4+OOPj3HqjGk7VWVvZS1fFFeypbiCL4or2FZS1aSE4Q8c3DjdM9lHn/QkctIS6OuLJTk+lsT4WJJ87tUzJZ6ctASyU+PJSU0gJy3hsCWHRJ9rs+jVI9o57l4sqHQwWVlZnHHGGYwZM4akpCR69+7duG/mzJk88sgjjBo1ihEjRjB58uR2TKnpjuqC9WwrqSJvdwV5u/e79+IKSqvqiI+NwRcbgy9O3HtMDPXatCtssL6eovKmjdfxcTEMykymd49EBg9OISctgZzUBLLT4undI5F+6Un0SU8Mq0HatL+orlEvIjOB3wOxwGOqel+z/YOAx4EcYC9wnaoWePt+DVyAG/W/ELgFSAXeD7lELvCUqt4qIjcA/wcUevseUtXHDpe+iRMnavNFutavX8+oUaOOPrOdVHfLrzlYXbCefZW17K2qZW9lLfsq69hb6WdPRW2TRuuiclf1FFrL1C89kaG9UslJTaCuXqkL1FMXdN1g64L1Xo+lmMZeS7ExQmZKPMflpDI0J4WhOan0y0giNqZ7t0N0NiKyXFUntrQvaiUVEYkFHgbOBQqApSIyX1U/DznsfuBJVX1CRKYD9wLXi8jpwBnAWO+4D4CpqroEGBdyj+XAv0Ku95yqfi9aeTKmM1FVKmuD7KuspaSyln2VtRTv91NQWk2BN16iYF8Vu8praKE5AsBVJaUl0rtHAqP6prnSRFYKw3qlMrRXKqnHqJuq6Tyi+RcxCchT1S0AIvIscAkQGlRGAz/0Pi8GXvI+K5AIxAMC+ICi0IuLyPFAL5qWXIzplmrqgqzbUcbq/DLWFJSypqCMgtLqxh5PoWIE+vRIJDczmclDs8jNSCKnRyJZKfH0TI4nMyWenik+eiaH12BtTKhoBpX+QH7I9wLg1GbHrAYux1WRXQakiUiWqn4kIouBnbig8pCqrm927mxcyST0N9YVInIWsAn4garmNzsHEZkDzAEYOHBgqzNnzLFSG6hn+95K8nZXsrOsmtKqOkqraimtrmNfVR27y2vYvLuisfdTnx6JjM1N59zRvb0AEe8CRko82SkJ9ElPJD7OgkWnFaiFfVshayjEHKadqT4IWxbD1g9AYiA2HmLiINbnPvc7GQY2fyS3XXuXXW8DHvLaQ97DtYcERWQYMArXZgKwUESmqGpoqWQ2cH3I91eAuarqF5FvA08A05vfUFUfBR4F16YS4fwYExZVN/o6b3cFReU1jdNzVHtTdFT5A2wtqWJLcQXb9lYd1F22R2IcPVPiyUjy0S8jiXNG9WZsbjonDcigd4/oTL9hoqi2CorWgS8RUnpBSvaBgFFfD7s/hy1L3Gvbv6GuCpIy4fiZMPJ8GDod4r3hB0Wfw+pnYM0/oWIXiHcdDTa955k/6HRBpRAYEPI9lwON6ACo6g5cSQURSQWuUNVSEfkW8LGqVnj7XgdOw6vqEpGTgDhVXR5yrZKQSz8G/DriOTKmFfbX1LG2sJzPCkvZXOR6S32xu4Lyw4zSTvTFMDAzmRF90jj/xL4M7ZXCsJw0+vdMIj3J130btuuDULUXUnOie5+6GljxBGx6A4bOgJNmuwf9oVSWwN4vIPO4wx8Xenz+x7DtQ9j+MexcBfUhfw8SA8lZLsBU7obKYrc9+3gYfx30PgG2/hs2vuYCSFwiDJkK+3fCrjWuRDL8PJfu48+DuAQXnOrrIFjn3mOis+JrNIPKUmC4iAzBBZPZwDWhB4hINrBXVeuBO3A9wQC2A98SkXtx1V9Tgd+FnHo1MLfZtfqq6k7v68VA8+oyY6LKHwhSuK+a/H3VbCmuYE2Ba9/YsqeShkranLQEhuakcPG4fgzNSWVYL9f7KTk+lkRvrqeEuBhiumvQOJy6Gnj+67D5TRh4Goz9GpxwKST1PPjY+iCU5EHxRijdDmX57r003z2gB50GJ1wOw88FX8i8eHXVsPwJ+OC37ld++kD44h14+xcwYhaMvx6GzQDEBYLNC2HzW1C4HNcUjCtBZB8POce790ANlO+A8p2wf4f73BAkYuOh/wQ4/b+g/0QXWCqLoWK3CyYVxdBnDBw3zQWN9P4H0jrhBhcgtn8EGxa4f5fEDJj1axhzxcHBLSYGYhJcgImiaHcpPh8XDGKBx1X1HhG5G1imqvNF5Epcjy/FVX/d7FVfxQJ/BM7y9r2hqj8Mue4W4HxV3RCy7V5cMAnguid/N3R/Szpil+LWTn0P8Lvf/Y45c+aQnJwc9jntnd/OorC0mpXb91FaVUd5TR37awKNo7GLymrYvreKov01hP7v1CstgbG5GYzNTWdsbjon9k8nKzW6/0N3SLVVsGOF+0We/6n7NR2sg2DtgfeEVJh5n3vIt3iNSph7NXz5Hpx8vbvWnk3uoXz8TDjxSnefnatgxyrY9RnUVR44Pz4NMgZA+gBI7OECRVUJxKfCiPNhzOWwb9uBYDLoTJh2OwyZArvXw8qnYPVcd05aP/dLv7IYEOh/Mgz/CvQZ69o69myE4k3uvcqrQEnKhB79IK2ve88cAgMmQ7/xrsqrkzlcl+KoBpWOriMGla1bt3LhhReydu3aoz63Yabi7Owwit+e9s5vR1XhD/DJlhLe37yH9zYXs6W4ssl+X6yQlugjLTGO3mmJ5GYmefNDJTMwM5nBWcn06ixtGzXlrj7+cI2+h7Nns2sMDtZCwA9Bv2tMrimDwmWwc/WBqp2ckdBzsAsGjS+fCzbF690v9ul3QVx80/Q9cxXkfwKX/slV6ajCjpWw5jn4bB5U7XHH+pKhz4nQdxz0Gwe9RkPPQe4XfOjI+mAAtr4Ha/8F61+BmlK3PTSYNBeoddVha55z1U3Dv+JKLYer7qreB3FJnTJwHI4FlUPoiEFl9uzZvPzyy4wYMYJzzz2XXr168fzzz+P3+7nsssv45S9/SWVlJVdddRUFBQUEg0HuvPNOioqKuO222xgxYgTZ2dksXrw4rPu1d37bk6ryRXEl20oqm6x1kb+vik1F+6kLKom+GCYfl8WZw7KZfFwWvdISSEt0U5R3mIkDVZs+MMNRvsM9TD9/2dXrp/WBcde6+vrMIUc+v+G+n/wZFt7lAkkT4h7wfU9yjcEDJsOASZCc2fK16qrhrZ/B0sdcr6QrH3fpqNoLT13h2gmueAxOuOzgc4N1LuAkZ7nqpqMNjoFaF2DiU2GgzVIRjnYZ/NglvH67K0ZHUp8TYdZ9h9wdOvX9W2+9xbx58/j0009RVS6++GLee+89iouL6devH6+99hrg5gRLT0/ngQceYPHixUdVUuluqmoD/DuvhHc27GbJxt3sLDswIWdCXAy5Pd1MtFOG53DW8GwmDO5JQkecObA+6Kpwlv8dNr0J8cmQnO1+NSdnuVdiunuwxye7d1+y+0W+/lXXSAzul/yUH8KutfDBA/D+/TDkLBj/dRh10aF/YVfshpdugryF7hf7efe6to24eIhNcKWPowl0viS44Deu7eDlm+GRKfCV/w+W/tVVI33tKdem0ZJYHww+8yj+8ZqJi4dh57T+fNOEBZUO7K233uKtt95i/PjxAFRUVLB582amTJnCj370I37yk59w4YUXMmVKC0X1bigQrGfF9lI+2FxMhT+Ieg2nDYXxL/dU8tGWEmoD9aTExzJleA63zMjh+D5p5PZMIic14dClD1VXX96j36EbOlWhcAWsfQEKPvWqgupCetwEDtStp/d37z1yISXHXTMu0T3g4hLd9/hUSEhrer/yna5+f8WTULbdBZIJN7gHeFUJVO5xDdKFK8C/33U9pVltRO8TYfrPYNQlrjG5QVmh60m04h/wrxvd/YdMhWHT3UO352B33Ka34OWb3PXPvx9OufHoS0qHMuoiV7p54UZ49VZXdXTNc67LrOkULKgczmFKFMeCqnLHHXfw7W9/+6B9K1asYMGCBfzsZz9jxowZ3HXXXe2QwvZXXlPHuxuLeWfDbhZv3E1pVR2xMUJyw+SDcuAtOy2B604dxIxRvThlcGb4AwBVYcFtrmomLtH10hl0uutBlDsJ9n3pAsnaf0HpNtdVc+Bk98CPjXPfY+NdtUxVCZQXQsFSqN4b3v1jfC64JKS6B78G3cP+K3fDiAuatj+0lPa6au9V6bqa9ujX8rHp/eGsH8OZP3LVQetegi8WuW6rAFnDXPXSxgXQ6wT4xivQKwpVpxkD4YYFsOyvrmdUbou1LKaDsqDSwYROfX/eeedx5513cu2115KamkphYSE+n49AIEBmZibXXXcdGRkZPPbYY03O7YrVXztKq9lUtJ/te6vYuqeK7Xsr2VpSxdY9lQTqlZ7JPqaP7MWMkb2Zcnw2PRIj2Ad/0d0uoIy/HhJ6wPYPXTXRe/W4cKVugNlx02DqT2DkBZCUceTr1lW7to2qEleqCfhd99Og33Wfra0Ef7krEdRWuAbrMbmu3SNraHhpF3HVX/HJQNYRDwdc19PjprmXquuam7cI8t52va5O/S6c84voNj7HxsGpB/+YMh2fBZUOJnTq+1mzZnHNNddw2mmnAZCamspTTz1FXl4eP/7xj4mJicHn8/GnP/0JgDlz5jBz5kz69esXdkN9R1VVG+DjLSW8t+ng3ldJvlgGZSUzNCeFi0amc+YJAxk3oGf4AwLLd7ixBZvecl1QT/6GG13c0i/+D37r2homfBMu/O2Bap6aclfFlf8ppPaG0ZeEN+gtlC/JBYdwA0R7EIHs4e41+TvtnRrTCVjvrw7W++tY6wj53VdZ2zjKPG93Bet2lLNs297G3lenDsliyvBsxuZmMDgrmZy0BKSuCl78tuvBlJjhRjJnDoGeQ1zdvy/Ja0zRA+8lea5Re9cad+P0Ae6cL99zDdYXPwS5Ew4kbOlf4bUfwpgr4fJHW9/l1pguxnp/mQ5jb2UtK7fvY8X2fazcXsqGXfvZW1nbuD8hLoZhvVL55hlDOGt4DhMH9zx4cabynTB3tgsOp37XjY/Y96Ub1bzupYPnOGogMTDgVJjxczd1Ra/R7pf4xjfg1R/AX8+ByTfB2T+FDa/Baz9yA+sue8QCijFhsqBiom7l9n08/cl2lm/bx5d7XDVWbIwwqm8aXxnd263NkZPK8PQgfas2EVtVBENPaHlMw661biBcdSnMngsjZjbdH6yDsgL3LgLIgSqr5KyW2zpGzHQN72//HD56yI3dKN/huql+9e+uy6oxJiwWVFqgqh1nYFsURbPqU1X5aEsJDy/O4995JfRIjOPU47K4auIATh6YwdjcDJJ2r4Qtb8KO1bB8jes51SDG58Y/jL3KlRZ8iW6epX/e4BrL/+N11/W0uVhf+IP3QiX2cG0mY66AV25xvY6untt0XihjzBFZUGkmMTGRkpISsrKyunRgUVVKSkpITGxDD55gwFUpxRzomquqLN64m4feyWPF9lJy0hL47/NHcu2pg0hJiHPtG1vfh2d+7d7BtYf0Gw8TvuECRWIGrHvRTb+x8TVISIfjznJVUr3HuHELh+oW21aDz4TvLXPpjAmzy7ExppE11DdrqK+rq6OgoICamppDnNV1JCYmkpubi8/XiuqdwuUEn7yCWP8+6mKSqI5JokITKQsmsDOYzvb44Qw+8TQmnzmDxKxB7py8RfDe/7nR3Km94Yxb3NQgh+p+Wx+EL9+F1c+5sRFDzoLL/uzGaxhj2o3N/XUILQUVc3hVtQEWfbyCM5ZcRWUgjhfqp5BCDTnxteQkBMj01dJPi+lR8QWi3lK2SZmufaQkz40gP/NWN+bjaMY5tGZ+K2NMVFjvL9Nmn+8oZ+6n23lz5Rf8Xe8kIcbP4tP/wowxkxiSk0JqQrM/pYaV7HaucrPUlm5zM9CedM3hR4AfigUUYzoFCyrmkHaUVvPqmh28vGoH63aUkxgH8zL+zKjKfLjmn1wx/DCT8MUnw4BT3MsY021YUDFN7KusZcHanby8ageffunmpho3IINfXDSa2aWPkrj0324SwcMFFGNMt2VBxTR2/33uky/55obvMIZ6fIkncf3JZ3HSmTMZ2K+fm2J96R9h0hyY9K32TrIxpoOyoNKNlVT4eWFFAXM/zefLPZVclLiGcTF51GSOZGzZa8jnL8L6H7luvLs/h6Ez3LoZxhhzCBZUuqGCfVU8vDiPecsLqAsqpwzuyfdnDOPiDc9AQQ6JN3/g1v4oWApb/+2Wih0wGb76Nzd7rDHGHII9IbqRwtJqHnonj3nL8xGE2acM5OunDWJ47zSoLIFX3nTTjcf63GvIWe5ljDFhsqDSDRRt38Qjn+7jqZUljcHkprOH0jc9ZAqSz/7pVig86er2S6gxptOLalARkZnA74FY4DFVva/Z/kHA40AOsBe4TlULvH2/Bi4AYoCFwC2qqiKyBOgLVHuX+Yqq7haRBOBJYAJQAnxNVbdGM38dVm0VbP2AwKaFVKx9nd41+ZxTP4baCX/i5unD6ZfRwnxWq552U6T0GXPs02uM6TKiFlREJBZ4GDgXKACWish8Vf085LD7gSdV9QkRmQ7cC1wvIqcDZwBjveM+AKYCS7zv16pq86Hw/wnsU9VhIjIb+F/ga1HIWse1dwss+DF8+T4E/QSIZ3nwBFIyRnLG/oWccUIhZIw9+Lxdn7lp5Gf937FPszGmS4nmjHmTgDxV3aKqtcCzwCXNjhkNvON9XhyyX4FEIB5IAHxA0RHudwnwhPd5HjBDuvKMkM2V74AnL6E+fxkLUy/iuto7uCTlaXzX/5PJt86FrOHw1p1uSvjmVj3jZgU+8cpjn25jTJcSzaDSH8gP+V7gbQu1Grjc+3wZkCYiWar6ES7I7PReb6rq+pDz/iYiq0TkzpDA0Xg/VQ0AZbSwKLeIzBGRZSKyrLi4uG057Ciq9hJ88lL8+0u4qurH/NfeKznt3CuZ/4MZnHV8jmt0P/duKNnsxpuECtTCmudgxKyW1y8xxpij0N5ze98GTBWRlbjqrUIgKCLDgFFALi5YTBeRKd4516rqicAU73X90dxQVR9V1YmqOjEnJydS+Wg3tZVlFD9yIYE9W/hG9Q/oM+p0Fv1oGjefPYyEuJDVCkfMgsFTYPGv3AJXDTa/BVUlMP66Y594Y0yXE82gUggMCPme621rpKo7VPVyVR0P/NTbVoortXysqhWqWgG8Dpzm7S/03vcDz+Cq2ZrcT0TigHRcg33npuraSuqqm2yur1deXfEla39zAT3L1vNg5s+447vf4qFrTqZ/Sw3xInDePVC9D97/zYHtq56BlF5uYKMxxrRRNHt/LQWGi8gQ3AN/NnBN6AEikg3sVdV64A5cTzCA7cC3ROReQHClmN95wSJDVfeIiA+4EHjbO2c+8A3gI+BK4B3tzPP6q7pSxPu/gfxPIDberUY4+Ey29xjPHR/7uKHoPk6O/Yz1p93PbefdeORFxfqe5LoMf/IInPKf4EuBzW/C5O/aoEZjTERE7UmiqgER+R7wJq5L8eOquk5E7gaWqep8YBpwr4go8B5ws3f6PGA68Bmu0f4NVX1FRFKAN72AEosLKH/xzvkr8A8RycN1T54drbxFVX0QPn8J3v8tFH0G6QNde0jlHuq//ADee4CBBHkaIBbqZ/6aUZOPYi6uGXe6VRXf/gXknuJGzo+7NkqZMcZ0N7ZIV0dapOvL9+CVW2HvF5B9PJz5Q9cjK9bHv/P28N8vfkZJyR5uPX4v1/YtIKnfGDipFb2mF/8K3v1fV+2VngtzFkc+L8aYLssW6eoMygrgueshOQuuehJGXgQxMdTUBfnpC6t5Yf5whGUAAByySURBVEUBg7OSefTGszl9WHbb7nX692H5E1CxC6b9JDLpN8YYLKh0DPVBeOFbbgzJtf+ErKEABOuVW59dxRvrdnHTtKF8f8ZwEn2xR7hYGBJSYeavYNHdMOaKtl/PGGM8FlQ6gvfuh+0fwqWPNAYUVeVnL63ljXW7uOvC0fzHmUMie88xV1hAMcZEXHuPUzHbPoJ374MTr4KTDvQt+N3bm5n76XZumjY08gHFGGOixIJKe6reBy/cCBkD4YLfuLEkwD8+3sbvF23mqom5/Pi8Ee2cSGOMCZ9Vf7UXVZj/fddY/h9vQWIPABZ8tpO7Xl7LjJG9+NVlJx557IkxxnQgVlJpL8v/Duvnw/Q7IXcCAB9+sYdbn13FyQN78tA1JxMXa/95jDGdi5VUjrWAH95/wI2UP26a694L5O3ez7f/sZyBWcn89RsTSYqPQC8vY4w5xiyoHEvbPoJXboE9G2HMlXD+/0FMDHsq/Hzz70tJiIvhbzecQkZyfHun1BhjWsWCyrFQU+amRVn2uJt25dp5MPxct6suyJwnl7G73M+zcyYzIDO5fdNqjDFtYEEl2nasgrmzoaIITvseTLvDDT7EzTR82z9Xs2J7KX+89mTGD+zZzok1xpi2saASbW//3E3aeOMi6H9yk10PLNzEq2t28pOZIzn/xL7tlEBjjIkc614UTbs+gy1L4LSbDwoo/1yWz0OL8/jaxAF8Z+px7ZM+Y4yJMAsq0fThQxCfChO+2WRz/t4qfvriWs4YlsX/XDbGxqIYY7oMCyrRUlYIa+fB+OshKaPJrgcWbkIEfvPVcfhsLIoxpguxJ1q0fPpn0Hq3qmKI9TvLeWlVId88Ywh90hPbKXHGGBMdFlSioaYclv0NRl8CPQc12fXrNzaQlhDHd6cObafEGWNM9FhQiYaV/wB/OZz+X002f7KlhMUbi/nutGGkJ/vaKXHGGBM9FlQiLRiAj/8Eg86A/hMaN6sq//vGBnr3SOCG0we3X/qMMSaKLKhE2ucvQVm+G+gYYuHnRazYXsqt5xxv83oZY7osCyqRpAof/gGyhsHxMxs3B+uV/3tzI8dlp/DVCbntmEBjjImuqAYVEZkpIhtFJE9Ebm9h/yARWSQia0RkiYjkhuz7tYisE5H1IvKgOMki8pqIbPD23Rdy/A0iUiwiq7zXjdHMW4u2/Rt2rnKDHWMO/NP+a0UBm3dXcNt5I2w6e2NMlxa1J5yIxAIPA7OA0cDVIjK62WH3A0+q6ljgbuBe79zTgTOAscAY4BRgasM5qjoSGA+cISKzQq73nKqO816PRSlrh/bhQ5CcBSdd3bippi7Ibxdu4qTcdGaN6XPMk2SMMcdSNH82TwLyVHWLqtYCzwKXNDtmNPCO93lxyH4FEoF4IAHwAUWqWqWqiwG8a64AOkZ9UjAAXyyCsV8DX1Lj5nnLC9hRVsNPZo60kfPGmC4vmkGlP5Af8r3A2xZqNXC59/kyIE1EslT1I1yQ2em93lTV9aEnikgGcBGwKGTzFV5V2jwRGdBSokRkjogsE5FlxcXFrc3bwUryIFgLfU9qsvmNtbsYmpPC6cOyI3cvY4zpoNq7gv82YKqIrMRVbxUCQREZBozClUL6A9NFZErDSSISB8wFHlTVLd7mV4DBXlXaQuCJlm6oqo+q6kRVnZiTkxO5nBStde+9xzRu2l9TxydflnDOqN6Ru48xxnRg0QwqhUBoaSHX29ZIVXeo6uWqOh74qbetFFdq+VhVK1S1AngdOC3k1EeBzar6u5Brlaiq3/v6GDCBY6loLcTEQfbxjZve37yHuqAyw4KKMaabiGZQWQoMF5EhIhIPzAbmhx4gItki0pCGO4DHvc/bcSWYOBHx4Uox671z/gdIB25tdq3QBUkubjj+mClaB9kjIO7AUsBvry8iI9nHyQMzDnOiMcZ0HVELKqoaAL4HvIl7wD+vqutE5G4Rudg7bBqwUUQ2Ab2Be7zt84AvgM9w7S6rVfUVr8vxT3EN/CuadR3+vtfNeDXwfeCGaOWtRUXroM+Bqq9gvbJkYzHTjs+xbsTGmG4jqis/quoCYEGzbXeFfJ6HCyDNzwsC325hewHQYhcqVb0DV9o59qr2Qnkh9D6hcdPK7fvYW1lrVV/GmG7FfkJHQtE69x4SVBZt2E1cjDB1RAQ7AxhjTAdnQSUSGoPKgeqvReuLmDQkkx6JNhuxMab7CCuoiMi/ROSCkEZ1E6poLSRnQ6qr6srfW8Wmogqr+jLGdDvhBok/AtcAm0XkPhEZEcU0dT5Fa13Vlzdi/u31RQDMGNmrPVNljDHHXFhBRVXfVtVrgZOBrcDbIvKhiHzT6/LbfdUHYff6JlVf72zYzdCcFAZnp7Rjwowx5tgLuzpLRLJw3XRvBFYCv8cFmYVRSVlnsXcLBGoauxPvr6nj4y02it4Y0z2F1aVYRF4ERgD/AC5S1Z3erudEZFm0EtcpNE7P4np+2Sh6Y0x3Fu44lQcbZgduTlUnRjA9nc+utSCxbjQ9rj0lPclG0Rtjuqdwq79Ge7MCAyAiPUXkpiilqXMpWgfZw8GX2DiK/uwRNoreGNM9hfvk+5Y30SMAqroP+FZ0ktTJFK1rbKRflW+j6I0x3Vu4QSVWQlaY8lZ1jD/M8d1DdSmUbW9sT3l7vY2iN8Z0b+G2qbyBa5T/s/f929627m335+7dK6ks3rCbUwbbKHpjTPcVblD5CS6QfNf7vhC3Zkn31mzOrx2l1Uw+LqsdE2SMMe0rrKCiqvXAn7yXaVC0FpJ6Qo9+qCpVtUFSEmLbO1XGGNNuwh2nMhy4F7eOSWLDdlU9Lkrp6hx2rXVVXyL464IE6pXk+KiuJmCMMR1auA31f8OVUgLA2cCTwFPRSlSnUF/v2lS8qq9KfwCA1AQLKsaY7ivcoJKkqosAUdVtqvoL4ILoJasT2Pcl1FU1NtJX+oMApFhQMcZ0Y+E+Af3etPebReR7QCGQGr1kdQLNpmeprG0oqVibijGm+wq3pHILkIxb+30CcB3wjWglqlMoWgcSAzkjgQPVX9amYozpzo74BPQGOn5NVW8DKoBvRj1VnUHROsgcCvHJAFR4QcWqv4wx3dkRSyqqGgTOPAZp6VyK1jZOdw8H2lSsod4Y052FW/21UkTmi8j1InJ5w+tIJ4nITBHZKCJ5InJ7C/sHicgiEVkjIktEJDdk369FZJ2IrBeRBxumiRGRCSLymXfN0O2ZIrJQRDZ77z3DzNvRqymHfVsb21PgQJuKjVMxxnRn4QaVRKAEmA5c5L0uPNwJXrXZw8As3PiWq0VkdLPD7geeVNWxwN24sTCIyOnAGcBYYAxwCjDVO+dPuMksh3uvmd7224FFqjocWOR9j47d691779CSihdUrE3FGNONhTuivjXtKJOAPFXdAiAizwKXAJ+HHDMa+KH3eTHwUsMtcYEsHhDABxSJSF+gh6p+7F3zSeBS4HXv2tO8858AluCml4m8yt2QkN5yULHqL2NMNxbuiPq/4R70TajqfxzmtP5Afsj3AuDUZsesBi7HLU18GZAmIlmq+pGILAZ24oLKQ6q6XkQmetcJvWZ/73PvkBUpdwEtzj8vInOAOQADBw48TPIPY9RFMLJpQa3CHyQ+Nob4OFtHxRjTfYX7BHwVeM17LQJ64HqCtdVtwFQRWYmr3ioEgiIyDBgF5OKCxnQRmRLuRVVVaSEIevseVdWJqjoxJ6cNU9SLuJenqjZg7SnGmG4v3OqvF0K/i8hc4IMjnFYIDAj5nuttC73uDlxJBRFJBa5Q1VIR+RbwsapWePteB04D/uFdp6VrFolIX1Xd6VWT7Q4nb5FS4Q/YGBVjTLfX2rqa4UCvIxyzFBguIkNEJB6YDcwPPUBEsr2R+gB3AI97n7fjSjBxIuLDlWLWe9Vb5SIy2ev19XXgZe+c+RwYkPmNkO3HRKU/YN2JjTHdXlhBRUT2i0h5wwt4hSM0gqtqAPge8CawHnheVdeJyN0icrF32DRgo4hswrWB3ONtnwd8AXyGa3dZraqvePtuwq3lkucd87q3/T7gXBHZDJzjfT9mKv027b0xxoRb/ZXWmour6gJgQbNtd4V8nocLIM3PC+IWBWvpmstw3Yybby8BZrQmnZFQ4Q+QlmglFWNM9xZuSeUyEUkP+Z4hIpdGL1mdT1VtwMaoGGO6vXDbVH6uqmUNX1S1FPh5dJLUObnqLwsqxpjuLdyg0tJx9gQNUeEP2LT3xphuL9ygskxEHhCRod7rAWB5NBPWmagqlf6AlVSMMd1euEHlv4Ba4DngWaAGuDlaiepsaoP1BOrVgooxptsLt/dXJdGcoLGTa1xKON6qv4wx3Vu4vb8WikhGyPeeIvJm9JLVudhkksYY44Rb/ZXt9fgCQFX3ceQR9d1Gw6qPNqLeGNPdhRtU6kWkcUpfERnMISZs7I6qaq2kYowxEH634J8CH4jIu7ip6KfgTR9v3LT3YKs+GmNMuA31b3hrmcwBVuIW06qOZsI6E2tTMcYYJ9xFum4EbsFNNb8KmAx8hFteuNursKWEjTEGCL9N5RbcOvHbVPVsYDxQevhTuo8qa6g3xhgg/KBSo6o1ACKSoKobgBHRS1bnUlnr2lSSrU3FGNPNhfvTusAbp/ISsFBE9gHbopeszqXCH8AXKyTEWVAxxnRv4TbUX+Z9/IWILAbSgTeilqpOxub9MsYY56ifhKr6bjQS0plV+oPWSG+MMbR+jXoTwpVUrOrLGGMsqERAZa1VfxljDFhQiQi3QJcFFWOMsaASAVXWpmKMMUCUg4qIzBSRjSKSJyIHrcciIoNEZJGIrBGRJSKS620/W0RWhbxqRORSb9/7Idt3iMhL3vZpIlIWsu+uaOYtVIU/YGNUjDGGKK4zLyKxwMPAuUABsFRE5qvq5yGH3Q88qapPiMh04F7gelVdDIzzrpMJ5AFvAajqlJB7vAC8HHK991X1wmjl6VAqa636yxhjILollUlAnqpuUdVa3DLElzQ7ZjTwjvd5cQv7Aa4EXlfVqtCNItIDN/fYSxFNdSvYOBVjjHGiGVT6A/kh3wu8baFWA5d7ny8D0kQkq9kxs4G5LVz/UmCRqpaHbDtNRFaLyOsickJLiRKROSKyTESWFRcXh5uXQ6oN1FMXVCupGGMM7d9QfxswVURWAlOBQiDYsFNE+gInAi0tXXw1TYPNCmCQqp4E/IFDlGBU9VFVnaiqE3NyctqcgYZp75NtfXpjjIlqUCkEBoR8z/W2NVLVHap6uaqOxy0ERuiyxcBVwIuqWhd6nohk46rXXgu5VrmqVnifFwA+77ioqrC1VIwxplE0g8pSYLiIDBGReFw11vzQA0QkW0Qa0nAH8HizazQvjTS4Eni1YeZk71p9RES8z5NweSuJSE4Oo7LWpr03xpgGUQsqqhoAvoeruloPPK+q60TkbhG52DtsGrBRRDYBvYF7Gs4XkcG4kk5Lc4211M5yJbBWRFYDDwKzVVUjlqFDsFUfjTHmgKg+Cb1qqAXNtt0V8nkeMO8Q527l4Ib9hn3TWtj2EPBQ61PbOpUN69Nbm4oxxrR7Q32nZyUVY4w5wIJKG1XYUsLGGNPIgkobWUnFGGMOsKDSRg3r09t6KsYYY0GlzSr9AeJihPhY+6c0xhh7ErZRw7xf3hAZY4zp1iyotFGFP2iN9MYY47Gg0kZVtbY+vTHGNLCg0kYV/gDJtuqjMcYAFlTarNLWpzfGmEYWVNqo0h+06i9jjPFYUGmjylpb9dEYYxpYUGmjSn+AFGtTMcYYwIJKm7nqLwsqxhgDFlTapDZQT22wnlRrUzHGGMCCSptU1dpkksYYE8qCShs0rk9vbSrGGANYUGmTxlUfraRijDGABZU2aSypWJuKMcYAFlTapKFNxUbUG2OMY0GlDRpWfbS5v4wxxolqUBGRmSKyUUTyROT2FvYPEpFFIrJGRJaISK63/WwRWRXyqhGRS719fxeRL0P2jfO2i4g86N1rjYicHM28gZv2HqykYowxDaL2NBSRWOBh4FygAFgqIvNV9fOQw+4HnlTVJ0RkOnAvcL2qLgYagkUmkAe8FXLej1V1XrNbzgKGe69TgT9571FTaW0qxhjTRDRLKpOAPFXdoqq1wLPAJc2OGQ28431e3MJ+gCuB11W16gj3uwQXoFRVPwYyRKRv65N/ZJU2TsUYY5qIZlDpD+SHfC/wtoVaDVzufb4MSBORrGbHzAbmNtt2j1fF9VsRSTiK+yEic0RkmYgsKy4uDj83Laj0B4iNERLirGnKGGOg/RvqbwOmishKYCpQCAQbdnoljROBN0POuQMYCZwCZAI/OZobquqjqjpRVSfm5OS0KfGV/iAp8bG2Pr0xxniiWW9TCAwI+Z7rbWukqjvwSioikgpcoaqlIYdcBbyoqnUh5+z0PvpF5G+4wBTW/SKtwhboMsaYJqJZUlkKDBeRISISj6vGmh96gIhki0hDGu4AHm92jatpVvXV0E4irnhwKbDW2zUf+LrXC2wyUBYSgKKiytZSMcaYJqL2RFTVgIh8D1d1FQs8rqrrRORuYJmqzgemAfeKiALvATc3nC8ig3Elj3ebXfppEckBBFgFfMfbvgA4H9dTrAr4ZnRydkCFP0iyBRVjjGkU1Seiqi7APexDt90V8nke0LxrcMO+rbTQ0K6q0w9xvBISlI4Ftz69dSc2xpgG7d1Q36nZqo/GGNOUBZU2sIZ6Y4xpyoJKG1TV2lLCxhgTyoJKG1T4AyRbm4oxxjSyoNJKdcF6agP1pFqbijHGNLKg0koHJpO0oGKMMQ0sqLRSZa1Ne2+MMc1ZUGmlxgW6rE3FGGMaWVBppQqr/jLGmINYUGmlhpKKVX8ZY8wBFlRaqdJbSthG1BtjzAEWVFrJlhI2xpiDWVBpJVtK2BhjDmZBpZUqrE3FGGMOYkGllar8QVuf3hhjmrEnYitV+AMk2/r0xhjThAWVVqq0ae+NMeYgFlRaqdLWpzfGmINYUGmlSr+tpWKMMc1ZUGklt5SwjVExxphQFlRaqcJv1V/GGNNcVIOKiMwUkY0ikicit7ewf5CILBKRNSKyRERyve1ni8iqkFeNiFzq7Xvau+ZaEXlcRHze9mkiUhZyzl3RzFtlrTXUG2NMc1ELKiISCzwMzAJGA1eLyOhmh90PPKmqY4G7gXsBVHWxqo5T1XHAdKAKeMs752lgJHAikATcGHK99xvOU9W7o5Q1wI1TsSlajDGmqWiWVCYBeaq6RVVrgWeBS5odMxp4x/u8uIX9AFcCr6tqFYCqLlAP8CmQG5XUH0GFP2CTSRpjTDPRDCr9gfyQ7wXetlCrgcu9z5cBaSKS1eyY2cDc5hf3qr2uB94I2XyaiKwWkddF5ISWEiUic0RkmYgsKy4uDj83IQLBevyBemtTMcaYZtq7of42YKqIrASmAoVAsGGniPTFVXO92cK5fwTeU9X3ve8rgEGqehLwB+Cllm6oqo+q6kRVnZiTk9OqRDdOe29BxRhjmohmUCkEBoR8z/W2NVLVHap6uaqOB37qbSsNOeQq4EVVrQs9T0R+DuQAPwy5VrmqVnifFwA+EcmOYH4aNcxQnGptKsYY00Q0g8pSYLiIDBGReFw11vzQA0QkW0Qa0nAH8Hiza1xNs6ovEbkROA+4WlXrQ7b3EW8iLhGZhMtbSQTz06hxfXprUzHGmCaiFlRUNQB8D1d1tR54XlXXicjdInKxd9g0YKOIbAJ6A/c0nC8ig3ElnXebXfoR79iPmnUdvhJYKyKrgQeB2V5jfsTZtPfGGNOyqD4VvWqoBc223RXyeR4w7xDnbuXghn1UtcU0q+pDwENtSG7YrE3FGGNa1t4N9Z3SgVUfrU3FGGNCWVBphezUeGaN6UN2akJ7J8UYYzoUq79phQmDMpkwKLO9k2GMMR2OlVSMMcZEjAUVY4wxEWNBxRhjTMRYUDHGGBMxFlSMMcZEjAUVY4wxEWNBxRhjTMRYUDHGGBMxEqU5FzsFESkGtrXy9GxgTwST05F01bxZvjqfrpq3zp6vQara4oJU3TqotIWILFPVie2djmjoqnmzfHU+XTVvXTVfYNVfxhhjIsiCijHGmIixoNJ6j7Z3AqKoq+bN8tX5dNW8ddV8WZuKMcaYyLGSijHGmIixoGKMMSZiLKi0gojMFJGNIpInIre3d3raQkQeF5HdIrI2ZFumiCwUkc3ee8/2TGNriMgAEVksIp+LyDoRucXb3qnzJiKJIvKpiKz28vVLb/sQEfnE+5t8TkTi2zutrSEisSKyUkRe9b53lXxtFZHPRGSViCzztnXqv8VDsaBylEQkFngYmAWMBq4WkdHtm6o2+Tsws9m224FFqjocWOR972wCwI9UdTQwGbjZ++/U2fPmB6ar6knAOGCmiEwG/hf4raoOA/YB/9mOaWyLW4D1Id+7Sr4AzlbVcSHjUzr732KLLKgcvUlAnqpuUdVa4FngknZOU6up6nvA3mabLwGe8D4/AVx6TBMVAaq6U1VXeJ/34x5U/enkeVOnwvvq814KTAfmeds7Xb4ARCQXuAB4zPsudIF8HUan/ls8FAsqR68/kB/yvcDb1pX0VtWd3uddQO/2TExbichgYDzwCV0gb14V0SpgN7AQ+AIoVdWAd0hn/Zv8HfD/gHrvexZdI1/gAv9bIrJcROZ42zr932JL4to7AaZjU1UVkU7b71xEUoEXgFtVtdz9+HU6a95UNQiME5EM4EVgZDsnqc1E5EJgt6ouF5Fp7Z2eKDhTVQtFpBewUEQ2hO7srH+LLbGSytErBAaEfM/1tnUlRSLSF8B7393O6WkVEfHhAsrTqvovb3OXyBuAqpYCi4HTgAwRafiR2Bn/Js8ALhaRrbgq5enA7+n8+QJAVQu99924HwKT6EJ/i6EsqBy9pcBwr1dKPDAbmN/OaYq0+cA3vM/fAF5ux7S0ilcf/1dgvao+ELKrU+dNRHK8EgoikgSci2svWgxc6R3W6fKlqneoaq6qDsb9P/WOql5LJ88XgIikiEhaw2fgK8BaOvnf4qHYiPpWEJHzcfW/scDjqnpPOyep1URkLjANNxV3EfBz4CXgeWAgbmmAq1S1eWN+hyYiZwLvA59xoI7+v3HtKp02byIyFteoG4v7Ufi8qt4tIsfhfuFnAiuB61TV334pbT2v+us2Vb2wK+TLy8OL3tc44BlVvUdEsujEf4uHYkHFGGNMxFj1lzHGmIixoGKMMSZiLKgYY4yJGAsqxhhjIsaCijHGmIixoGJMJyUi0xpm8zWmo7CgYowxJmIsqBgTZSJynbcGyioR+bM3IWSFiPzWWxNlkYjkeMeOE5GPRWSNiLzYsMaGiAwTkbe9dVRWiMhQ7/KpIjJPRDaIyNMSOrmZMe3AgooxUSQio4CvAWeo6jggCFwLpADLVPUE4F3cTAYATwI/UdWxuNkAGrY/DTzsraNyOtAwu+144Fbc2j7H4ebQMqbd2CzFxkTXDGACsNQrRCThJg6sB57zjnkK+JeIpAMZqvqut/0J4J/evFH9VfVFAFWtAfCu96mqFnjfVwGDgQ+iny1jWmZBxZjoEuAJVb2jyUaRO5sd19r5kkLnwQpi/0+bdmbVX8ZE1yLgSm8djYZ1yQfh/t9rmH33GuADVS0D9onIFG/79cC73sqVBSJyqXeNBBFJPqa5MCZM9qvGmChS1c9F5Ge4Vf9igDrgZqASmOTt241rdwE3BfojXtDYAnzT23498GcRudu7xlePYTaMCZvNUmxMOxCRClVNbe90GBNpVv1ljDEmYqykYowxJmKspGKMMSZiLKgYY4yJGAsqxhhjIsaCijHGmIixoGKMMSZi/n92BcndgPjsQAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxcdb3/8ddnlkySSZqkSdp0X2lpbaGlpbQiUkC0LJeirAJeF66Fq3hxgQtcFZWr94p6ARdUUKooyiLLzwpF1rKvbYHSlaZ7uiXNvicz8/n98T1Jp+mkpG0mk+XzfDzmMTPnnJn5njSdd77L+X5FVTHGGGM68qW6AMYYY3onCwhjjDEJWUAYY4xJyALCGGNMQhYQxhhjErKAMMYYk5AFhDFHSUT+KCI/7OKxW0XkE0f7Psb0BAsIY4wxCVlAGGOMScgCwgwIXtPO9SKySkTqReQeERkqIk+KSK2IPCsieXHHnycia0SkSkReEJEpcftmishK73UPAukdPutcEXnXe+1rInLcEZb5yyJSLCIVIrJERIZ720VEbheRUhGpEZH3RWSat+9sEVnrlW2niFx3RD8wY7CAMAPLBcCZwCTgX4Angf8CCnH/F/4DQEQmAfcDX/f2LQX+ISJpIpIG/D/gz8Bg4G/e++K9diawGLgKyAfuApaISOhwCioipwP/C1wMDAO2AQ94uz8JfNw7jxzvmHJv3z3AVaqaDUwDnj+czzUmngWEGUh+qap7VXUn8DLwpqq+o6pNwGPATO+4S4AnVPUZVW0FfgZkAB8F5gJB4A5VbVXVh4G34z5jEXCXqr6pqlFVvRdo9l53OC4HFqvqSlVtBm4C5onIWKAVyAaOBURV16nqbu91rcBUERmkqpWquvIwP9eYdhYQZiDZG/e4McHzLO/xcNxf7ACoagzYAYzw9u3UA2e53Bb3eAzwLa95qUpEqoBR3usOR8cy1OFqCSNU9XngV8CdQKmI3C0ig7xDLwDOBraJyIsiMu8wP9eYdhYQxhxsF+6LHnBt/rgv+Z3AbmCEt63N6LjHO4AfqWpu3C1TVe8/yjKEcU1WOwFU9ReqOguYimtqut7b/raqLgSG4JrCHjrMzzWmnQWEMQd7CDhHRM4QkSDwLVwz0WvA60AE+A8RCYrIZ4A5ca/9HXC1iJzkdSaHReQcEck+zDLcD3xRRGZ4/Rf/g2sS2yoiJ3rvHwTqgSYg5vWRXC4iOV7TWA0QO4qfgxngLCCM6UBVNwBXAL8E9uE6tP9FVVtUtQX4DPAFoALXX/Fo3GuXA1/GNQFVAsXesYdbhmeB7wKP4GotE4BLvd2DcEFUiWuGKgd+6u37HLBVRGqAq3F9GcYcEbEFg4wxxiRiNQhjjDEJWUAYY4xJyALCGGNMQhYQxhhjEgqkugDdpaCgQMeOHZvqYhhjTJ+yYsWKfapamGhfvwmIsWPHsnz58lQXwxhj+hQR2dbZPmtiMsYYk5AFhDHGmIQsIIwxxiTUb/ogEmltbaWkpISmpqZUFyXp0tPTGTlyJMFgMNVFMcb0E/06IEpKSsjOzmbs2LEcOPlm/6KqlJeXU1JSwrhx41JdHGNMP9Gvm5iamprIz8/v1+EAICLk5+cPiJqSMabn9OuAAPp9OLQZKOdpjOk5/T4gPkxLJMae6iaaW6OpLooxxvQqAz4gIrEYpbVNNEWSs65KVVUVv/71rw/7dWeffTZVVVVJKJExxnTNgA8Iv9c0E0vSuhidBUQkEjnk65YuXUpubm5SymSMMV3Rr0cxdYXP5wVELDkBceONN7Jp0yZmzJhBMBgkPT2dvLw81q9fzwcffMD555/Pjh07aGpq4tprr2XRokXA/qlD6urqOOuss/jYxz7Ga6+9xogRI/j73/9ORkZGUsprjDFtBkxA/OAfa1i7qybhvvrmCGkBH0H/4VWopg4fxPf+5SOHPObHP/4xq1ev5t133+WFF17gnHPOYfXq1e3DURcvXszgwYNpbGzkxBNP5IILLiA/P/+A99i4cSP3338/v/vd77j44ot55JFHuOKKKw6rrMYYc7iS2sQkIgtEZIOIFIvIjQn2h0TkQW//myIy1tueJiJ/EJH3ReQ9EZmfzHIC9NTCq3PmzDngWoVf/OIXHH/88cydO5cdO3awcePGg14zbtw4ZsyYAcCsWbPYunVrD5XWGDOQJa0GISJ+4E7gTKAEeFtElqjq2rjDrgQqVXWiiFwK3IpbBP7LAKo6XUSGAE+KyImqesQ9yYf6S3/NzmrywmkMz01+s004HG5//MILL/Dss8/y+uuvk5mZyfz58xNeyxAKhdof+/1+Ghsbk15OY4xJZg1iDlCsqptVtQV4AFjY4ZiFwL3e44eBM8QN6J8KPA+gqqVAFTA7WQX1+YRokvogsrOzqa2tTbivurqavLw8MjMzWb9+PW+88UZSymCMMUcimX0QI4Adcc9LgJM6O0ZVIyJSDeQD7wHnicj9wChglnf/VvyLRWQRsAhg9OjRR1xQn0jSRjHl5+dz8sknM23aNDIyMhg6dGj7vgULFvDb3/6WKVOmMHnyZObOnZuUMhhjzJHorZ3Ui4EpwHJgG/AacNCVbKp6N3A3wOzZs4/4G97vgyRVIAD461//mnB7KBTiySefTLivrZ+hoKCA1atXt2+/7rrrur18xhiTSDIDYifur/42I71tiY4pEZEAkAOUq6oC32g7SEReAz5IVkF9krwmJmOM6auS2QfxNnCMiIwTkTTgUmBJh2OWAJ/3Hl8IPK+qKiKZIhIGEJEzgUiHzu1ulcwmJmOM6auSVoPw+hSuAZ4C/MBiVV0jIrcAy1V1CXAP8GcRKQYqcCECMAR4SkRiuFrG55JVTgC/T4i1WkAYY0y8pPZBqOpSYGmHbTfHPW4CLkrwuq3A5GSWLZ5PhKjVIIwx5gADfi4mAF+SO6mNMaYvsoDATdinqtYPYYwxcSwgSO6EfUc63TfAHXfcQUNDQzeXyBhjusYCAtcHASSlH8ICwhjTV/XWC+V6lN9brTOWhDWD4qf7PvPMMxkyZAgPPfQQzc3NfPrTn+YHP/gB9fX1XHzxxZSUlBCNRvnud7/L3r172bVrF6eddhoFBQUsW7as+wtnjDGHMHAC4skbYc/7CXeFYzHGt8ZIS/PD4aztXDQdzvrxIQ+Jn+776aef5uGHH+att95CVTnvvPN46aWXKCsrY/jw4TzxxBOAm6MpJyeH2267jWXLllFQUND1MhljTDexJiZAvFDQJHdSP/300zz99NPMnDmTE044gfXr17Nx40amT5/OM888ww033MDLL79MTk5OUsthjDFdMXBqEIf4S7+1NcrmvbWMHpxJbmZa0oqgqtx0001cddVVB+1buXIlS5cu5Tvf+Q5nnHEGN998c4J3MMaYnmM1CPZ3UidjmGv8dN+f+tSnWLx4MXV1dQDs3LmT0tJSdu3aRWZmJldccQXXX389K1euPOi1xhjT0wZODeIQvFGuRJPQSR0/3fdZZ53FZZddxrx58wDIysrivvvuo7i4mOuvvx6fz0cwGOQ3v/kNAIsWLWLBggUMHz7cOqmNMT1Okt3u3lNmz56ty5cvP2DbunXrmDJlyoe+VlV5f2c1QwelM3RQerKKmHRdPV9jjGkjIitUNeGCbNbEhOukthldjTHmQBYQHlsTwhhjDtTvA6KrTWh9fcK+/tJUaIzpPfp1QKSnp1NeXt6lL0+/SFLmYuoJqkp5eTnp6X23/8QY0/v061FMI0eOpKSkhLKysg89tqy2GYDGslCyi5UU6enpjBw5MtXFMMb0I/06IILBIOPGjevSsT/5w1uU1TXz+NdmJLlUxhjTN/TrJqbDEQ4FqG+OproYxhjTa1hAeLLTA9Q1R1JdDGOM6TUsIDzhtAD1FhDGGNPOAsITDgVoaInatRDGGOOxgPBkhVx/fX2L1SKMMQYsINqF2wLCmpmMMQawgGiXlW4BYYwx8SwgPFkhPwB1NtTVGGMAC4h24TSrQRhjTDwLCE9bH0RtkwWEMcaABUS7LOukNsaYA1hAeNo7qW2YqzHGAEkOCBFZICIbRKRYRG5MsD8kIg96+98UkbHe9qCI3Csi74vIOhG5KZnlhP01CJtuwxhjnKQFhIj4gTuBs4CpwGdFZGqHw64EKlV1InA7cKu3/SIgpKrTgVnAVW3hkSyhgA+/T6yJyRhjPMmsQcwBilV1s6q2AA8ACzscsxC413v8MHCGiAigQFhEAkAG0ALUJLGsiAjhND911kltjDFAcgNiBLAj7nmJty3hMaoaAaqBfFxY1AO7ge3Az1S1ouMHiMgiEVkuIsu7sijQh8kKBew6CGOM8fTWTuo5QBQYDowDviUi4zsepKp3q+psVZ1dWFh41B/q1oSwGoQxxkByA2InMCru+UhvW8JjvOakHKAcuAz4p6q2qmop8CowO4llBdxIJhvFZIwxTjID4m3gGBEZJyJpwKXAkg7HLAE+7z2+EHheVRXXrHQ6gIiEgbnA+iSWFWhrYrKAMMYYSGJAeH0K1wBPAeuAh1R1jYjcIiLneYfdA+SLSDHwTaBtKOydQJaIrMEFzR9UdVWyytrGFg0yxpj9Asl8c1VdCiztsO3muMdNuCGtHV9Xl2h7soVDARvFZIwxnt7aSZ0SWSG/NTEZY4zHAiKO66SO4rpBjDFmYLOAiBMOBYjGlOZILNVFMcaYlLOAiGPzMRljzH4WEHHaFg2yjmpjjLGAOEDYahDGGNPOAiJOdrotGmSMMW0sIOK01SBsug1jjLGAOEBWyA9gM7oaYwwWEAcI27rUxhjTzgIiTnsntY1iMsYYC4h47cNcrQZhjDEWEPH8PiEzzW9NTMYYgwXEQcIhWzTIGGPAAuIgti61McY4FhAdhEN+6ppaU10MY4xJOQuIDtyqclaDMMYYC4gOstNtXWpjjAELiINYJ7UxxjgWEB2EQwEb5mqMMVhAHMSNYrKAMMYYC4gOwmkBmlpjRKK27KgxZmCzgOggq31NCBvJZIwZ2CwgOmif8ts6qo0xA5wFRAc25bcxxjgWEB3YutTGGONYQHSQZWtCGGMMYAFxkLY1IayJyRgz0FlAdJCdbk1MxhgDSQ4IEVkgIhtEpFhEbkywPyQiD3r73xSRsd72y0Xk3bhbTERmJLOsbayT2hhjnKQFhIj4gTuBs4CpwGdFZGqHw64EKlV1InA7cCuAqv5FVWeo6gzgc8AWVX03WWWNF/aGuda32HUQxpiBLZk1iDlAsapuVtUW4AFgYYdjFgL3eo8fBs4QEelwzGe91/aIUMBP0C/WxGSMGfCSGRAjgB1xz0u8bQmPUdUIUA3kdzjmEuD+RB8gIotEZLmILC8rK+uWQoNrZrJRTMaYga5Xd1KLyElAg6quTrRfVe9W1dmqOruwsLDbPjfLZnQ1xpikBsROYFTc85HetoTHiEgAyAHK4/ZfSie1h2SyGV2NMSa5AfE2cIyIjBORNNyX/ZIOxywBPu89vhB4XlUVQER8wMX0YP9DG1s0yBhjIJCsN1bViIhcAzwF+IHFqrpGRG4BlqvqEuAe4M8iUgxU4EKkzceBHaq6OVll7Ew4FKC6sbWnP9YYY3qVpAUEgKouBZZ22HZz3OMm4KJOXvsCMDeZ5etMVsjPrqrGVHy0Mcb0Gr26kzpVsmwUkzHGWEAkYutSG2OMBURCWV4ntddfbowxA5IFRALhUICYQmOrTbdhjBm4uhQQInKtiAwS5x4RWSkin0x24VLFFg0yxpiu1yC+pKo1wCeBPNwEej9OWqlSrH1dauuoNsYMYF0NiLYJ9M4G/qyqa+K29TtZoSAA9c3WxGSMGbi6GhArRORpXEA8JSLZQCx5xUqttim/rYnJGDOQdfVCuSuBGcBmVW0QkcHAF5NXrNTKskWDjDGmyzWIecAGVa0SkSuA7+Cm5u77dq+Cu+fDzhXtm9pXlbP5mIwxA1hXA+I3QIOIHA98C9gE/ClppepJGbmw6x3YtX/BuiwbxWSMMV0OiIg3y+pC4FeqeieQnbxi9aCcUZCeA3veb9/UHhA2iskYM4B1tQ+iVkRuwg1vPcWbijuYvGL1IBEYOv2AgMhM8yNifRDGmIGtqzWIS4Bm3PUQe3CL//w0aaXqaUXToXQtxNywVhEhnBagzoa5GmMGsC4FhBcKfwFyRORcoElV+0cfBLiAaG2Aiv1LT4RDfqtBGGMGtK5OtXEx8BZu7YaLgTdF5MJkFqxHFU1393tWtW8KhwLU2SgmY8wA1tU+iG8DJ6pqKYCIFALPAg8nq2A9qvBY8AVdP8S0CwDIDgWosVXljDEDWFf7IHxt4eApP4zX9n6BNBcScR3VU4YN4t3tVbRG++0F48YYc0hd/ZL/p4g8JSJfEJEvAE/QYSnRPq9o2gEBMX/yEGqbI6zYVpnCQhljTOp0tZP6euBu4Djvdreq3pDMgvW4oulQtxfqXEXp5In5BP3Csg2lH/JCY4zpn7rcTKSqj6jqN73bY8ksVEq0d1S7WkR2epATxw7mhfVlKSyUMcakziEDQkRqRaQmwa1WRGp6qpA9Yug0dx/XzHTa5CFs2FvLzqrGFBXKGGNS55ABoarZqjoowS1bVQf1VCF7ROZgN+1GfEAcWwjAC9bMZIwZgPrPSKTuUHTglBsTCrMYmZfBMmtmMsYMQBYQ8YZOg/KN0OqalESE0yYP4dXifTRHbNoNY8zAYgERr2g6aMzNy+Q57dhCGlujvLWlIoUFM8aYnmcBEa/DSCaAeeMLCAV81sxkjBlwLCDi5Y6B0KADAiIjzc+8CfnWUW2MGXCSGhAiskBENohIsYjcmGB/SEQe9Pa/KSJj4/YdJyKvi8gaEXlfRNKTWVYAfD7XDxEXEOCGu27eV8/WffVJL4IxxvQWSQsIEfEDdwJnAVOBz4rI1A6HXQlUqupE4HbgVu+1AeA+4GpV/QgwH+iZmfOKpsPeNRDbPwfTaZOHADbc1RgzsCSzBjEHKFbVzaraAjyAW7I03kLgXu/xw8AZIiLAJ4FVqvoegKqWq2rPDCMqmgYtdVC5pX3T6PxMxheGWbbB+iGMMQNHMgNiBLAj7nmJty3hMaoaAaqBfGASoN4EgStF5D8TfYCILBKR5SKyvKysm768E3RUg6tFvL65nMYWG+5qjBkYemsndQD4GHC5d/9pETmj40GqereqzlbV2YWFhd3zyYVTQPwJA6IlEuP1zfu653OMMaaXS2ZA7ARGxT0f6W1LeIzX75CDW2uiBHhJVfepagNuavETkljW/YLpUDj5oIA4cVwemWl+G+5qjBkwkhkQbwPHiMg4EUkDLgWWdDhmCfB57/GFwPOqqsBTwHQRyfSC41RgLT2laDrsXX3AplDAz8kTC3h+fSmuiMYY078lLSC8PoVrcF/264CHVHWNiNwiIud5h90D5ItIMfBN4EbvtZXAbbiQeRdYqapPJKusBxk6DWp2Qn35AZtPmzyEnVWNvLujqseKYowxqZLUPghVXaqqk1R1gqr+yNt2s6ou8R43qepFqjpRVeeo6ua4196nqh9R1WmqmrCTOmnaOqr3HtjMdN6M4eRmBvn5cxt7tDjGGJMKvbWTOrU6GcmUFQpw1ccn8MKGMluK1BjT71lAJBIugOzhsHvVQbv+dd4Y8sNp3PHsBykomDHG9BwLiM6MOwU2PAmNB/Y3hEMBrj51Ai9v3GczvBpj+jULiM7M+yq01MLyxQftumLuGAqyQtz+jNUijDH9lwVEZ4YdDxNOhzd+A61NB+zKSPPzlfkTeH1zOa9tsgvnjDH9kwXEoZx8LdSXwnv3H7TrspNGM3RQiDue2WjXRRhj+iULiEMZdyoMmwGv/RJiB87BlB70c81pE3lrawWvFpd38gbGGNN3WUAcioirRVRsgvWPH7T74hNHMTwnnf97ZoPVIowx/Y4FxIeZuhDyxsErd0CHEAgF/Fxz+jG8s72KF2wqcGNMP2MB8WF8fvjo12DXStj6ykG7L5w1kjH5mXz7sfcpr2tOQQGNMSY5LCC6YsZlEC6EV39+0K60gI87LzuBffUtfO3+d4hEYwnewBhj+h4LiK4IZsBJV0HxM7Bn9UG7p43I4UfnT+O1TeX89KkNKSigMcZ0PwuIrpp9JQTD8NovEu6+aPYorpg7mrte2swTq3b3cOGMMab7WUB0VeZgmPUFeP9h2PxiwkNuPvcjzBydy/UPv8fGvbU9Wz5jjOlmFhCH45RvQsEkuO8CWP3IQbvTAj5+c/ksMtMCXPXnFdQ0taagkMYY0z0sIA5HuAC+9CSMnA0PXwlv/PagQ4py0rnzsplsq2jgmw++RzRm10cYY/omC4jDlZEHn3sMJp8N/7wBnv3+QddHnDQ+n++eM4Vn1+3lxkdWEbOQMMb0QYFUF6BPCmbAxX+Cpd+CV26HulL4l5+DP9h+yBdOHkdFQyu/eG4joaCP/144DRFJYaGNMebwWEAcKX8Azr0DsorgxR8DAuffecAh3/jEMTS3Rrnrpc2kB/x8+5wpFhLGmD7DAuJoiMBpN0GsFV7+P5h4Oky7IG63cONZx9IcifH7V7aQHvRz3acmp7DAxhjTdRYQ3WH+TW7o6z++ASPnQO6o9l0iws3nTqWpNcqvlhWTHvRxzenHpLCwxhjTNdZJ3R38Qbjgd6BReHTRQVOD+3zCjz49nc/MHMHPnv6A/1m6jpaITclhjOndLCC6y+DxcPbPYPtr8MptB+32+4SfXHgcV8wdzd0vbebC377Gln31KSioMcZ0jQVEdzr+UtcHsex/oWTFQbsDfh8/PH86v71iFtsrGjjnFy/zt+U7bC0JY0yvZAHRnUTgnNtg0Ah45EpoTjzdxoJpRTx57SkcNzKH6x9exX888C7VjUd51XXUrto2xnQvC4julpELn7kLqrbBP66FhoqEhw3LyeAv/zaX6z81maXv72bBHS/xz9V7jqw2sfEZ+N+RsOf9oyy8McbsZwGRDGM+Cqfe6OZr+tkkeOByWLsEIgcuKOT3CV89bSIPXz2PnIwgV9+3givvXc6Oioauf1YsBs/cDJEmePOubj4RY8xAJv2l/Xv27Nm6fPnyVBfjQLvfg1UPwft/g7q9kJ4DU8+H0fMgfwLkT3SzxAKt0Rj3vraV2575gGhM+drpE/nyx8cTCvgP/RmrHoJHv+w6yWt2w7fWu1qMMcZ0gYisUNXZCfdZQPSAaAS2vAirHoR1/4DWuBpCeq4LivGnwvz/YnddK//9+FqWvr+H8YVhfnj+ND46oaCT922FX82GtGxY+Eu4ez4suBXmXt0jp2WM6ftSFhAisgD4OeAHfq+qP+6wPwT8CZgFlAOXqOpWERkLrAPalmd7Q1UP+a3XqwMiXrQVKrdBeTFUbHL3ZR/Atlfg2HPhwsUQCLFsQynf+/satlc08JkTRvDts6eQnxU68L3evgee+CZc9hBM+hT87nRoroOvvuk6zI0x5kOkJCBExA98AJwJlABvA59V1bVxx3wFOE5VrxaRS4FPq+olXkA8rqrTuvp5fSYgOvPGb93ssONPg0v/Amlhd/X188Xc9dImwqEAN511LBfNGoXPJ9DaCD+fAXlj4EtPuUB45y/w96/A5x+Hcaek+oyMMX3AoQIimZ3Uc4BiVd2sqi3AA8DCDscsBO71Hj8MnCEDdTa7uVfDwl+7pqg/nQ+Nle1zNy39j1OYNDSbGx55n0vufp01u6rhrbuhbg+c8b39tYVpn3FNVsvvSe25GGP6hWQGxAhgR9zzEm9bwmNUNQJUA/nevnEi8o6IvCgiCf8cFpFFIrJcRJaXlZV1b+lTYeblcNG9sOsd+OO5bhpx4Jih2Ty4aC4/ufA4ikvruPQXT1P73E/ZV3QKsdEf3f/6YAbMuNz1c9TuTdFJGGP6i946zHU3MFpVZwLfBP4qIoM6HqSqd6vqbFWdXVhY2OOFTIqp58FlD0LFZli8ANY9Dq1NiAgXzx7Fsuvms3jym2THavnCtk9x+v+9wB9f3UJdc8S9fvaXIBaBd/6U2vMwxvR5yQyIncCouOcjvW0JjxGRAJADlKtqs6qWA6jqCmATMCmJZe1dJp7hVq1rroUHL4efHQOPXgUfPEVuZB8n7r6f2JSFXHXpBeSF0/j+P9Yy73+e4/tL1lAcGwrjToUV9x40aaAxxhyOZHZSB3Cd1GfgguBt4DJVXRN3zFeB6XGd1J9R1YtFpBCoUNWoiIwHXvaOS3xZMv2gkzqRaCtseQnWPOqajZqqQfyAwlfehEKXme9sr+SPr23lyff30BKNce2wtXyj8odELvkrgSnnpPYcjDG9WiqHuZ4N3IEb5rpYVX8kIrcAy1V1iYikA38GZgIVwKWqullELgBuAVqBGPA9Vf3HoT6rXwZEvEgLbF4Gax6DwmPhY18/6JB9dc08tHwHD7y+mb81fZli3ziWn/w7/nXeGPLCaSkotDGmt7ML5QaYaEzZ/si3GbPm13y8+Xb2BYq4ePYo/u1j4xmdn5nq4hljehELiIGoeifcMZ1IRj7VrX5aW5oJECHsj+LLHEzahXchY09OdSmNMSl2qICwJUf7q5wR8MkfEih5m3x/Go0xH+tKm1lT2sjJNSsZ9cfzWDbhPxn7ya8waWh2qktrjOmFrAYxwNQ1R/jn8nVMeulajmtewR8in+Khwf/OOTNGcubUIiYNzWKgXqtozEBkTUzmYNEIDUu/TeaK3/Je2kw+V/Pv1JBF0aB0Tp1UyKmTCzl5YgE5GcFUl9QYk0QWEKZz79wHj3+DSPYIVoz6Iq9UDOIfOzPZ2hTG7/Mxa0weF07L45zc7YT3vAFbX3VXeueNhdFz3dTlo+e651bzMKbPsYAwh7b9TXjoX93cTp5oMIuytJFUNUaZGN1EQGJE8VE7eDrZE07CX7UNdrzhrs0AyCqC8fPhuItg3HzwW/eWMX2BBYT5cNEIVO+A8k3eNORuKnKNNlOWO5On6idw9+Z8dtT7yc0MMmt0HlOHZTEnXMrUyFoG71uBFD/tAiNrKEy7EI67GIYdD9EWKF0Lu951tY89q6BgEnzi+zBoeKrP3JgBzQLCdItINMbLxft4YtVu3i+pprisjmjM/f5khwLMGJ7OhYPWcXL9c+TvWobEWmHQCDfpYKzVvUl6DgydBiXLwR+E+YruXtIAABQcSURBVDfCSVe7x8aYHmcBYZKiqTXKxr11rN1dzZpdNbyzvYo1u6qJKeRJHf82eBXzg2vIHDqBoZPnkjl29v6+ioot8OQNsPEpKJwC5/wfdOW6jFgM9rznrvMYPQ/C+Z0f21gJm54Hf5pbjMn6SIw5iAWE6TF1zRHe2V7J21sqeGtrBe9sr6I5EkMEPjJ8EPPG5zNvQj6zxgwmJz0AG5bCkzdC9XaYuhBGzXXraw8eB7ljIJgODRXui774WSh+DupLvU8TGHacW2RpwmnuteUbYePTsPEZ2PEmaMwdOvYUOPcOKJiYsp+NMb2RBYRJmeZIlHe3V/H65nJe31TOO9uraIm6L+3xBWGOG5nDzGEhPll+H0Ub/ow018S9WiC7COr2ui/6jDyYcAYccybkjoatr8CmZVDylpviXHz7A2HY8XDMJ92tdC08fTNEmuDU6+Gj10Igbm4qVdi3ETa/AKEsF1Rp4R77GRmTShYQptdoao2yclslK7dX8l5JNatKqthb0wyA3wcnFiqnFtYyM6uKiYFSBrfuxpc7GiaeCSNOAJ//4DdtrnXDb3e8AfkTYeInXLDEq90D/7zRm+xwCiz43/1NUJuWQU3J/mPTst1orBM+D8NnJPGnYUzqWUCYXm1vTRPv7ajivZIqVpVU8+6OKmqb3AJIWaEAU4cN4thh2RxbNIgpw7KZXJRNZtoRDqPd8CQ8cd3+QAjlwPhTYcLprpmqdi+s+KObYj3SBMNmuFX6iqa5/pOsIvD1gnW2VF0wph+0jpYxh8UCwvQpsZiypbye93ZU8e6OKtbuqmH9ntr2VfNEYMzgTCYNzebYomwmFw1iclEWY/PDBPxd+PJuroW1S6DgGBh+QuJrNhorYdXfXFiUrtm/3R9yzVt5Y13/SKQFos3uPtLkRmvFom4tj1jEPdaoG85beCwMmbL/PnuYd8LeMbGI++JPCyfuUG9tgm2vwAdPuVvVNpi0AE69wdWuuqpyG3zwT9j8orvIce6/J38U2d41kJYFeWOS+znmsFlAmD4vFlN2VjWydncN63fXsn5PDRv21rJ1Xz3eSFvSAj6mFGUzfWQOx43IZfrIHI4ZktW10OiMKlRucaOuKre6W9U2dx9pcX0Z/hAEQm60lD/NBY4vAL6guxdxX8pl66ChfP97x/eZxPOHIHuoC5DsIldrqdnpmsJa6yGQ4S5KzJ/groRvqnJ9LafeCCNnHfx+rU2wd7WrPX3wT/cY3PvX7oYhU+Gc22DMvCP/OXVm1zvwwq3wwZMQDMO5t8Pxl3T/58RrC+hgenI/p5+wgDD9VlNrlOLSOjbscaGxemcNq3dWU+vVNtKDPiYXDWJCYZgJhVnt96PzMwkFEvRnJFv9PihdB2XrXb+IL+D6VXx+97jtmNo97sr22j1Qsxsycl0ITFoA406BYIb3A6iBt+6G13/laj0TPwHDZ7pAqtrm7tuukBcfjP4oTF4Ak85yI7rWP+GGG1fvgBlXwJm3JB46HIsm7v/pzM6V8OKtLpDSc10tZfOLsP019zln/yQ5AwFKVsDfvwr1ZbDwVzD5rM6Prdnt+qU0Cqd/Fwond395+gALCDOgtDVRvV9SzXslVWzYU8umsrr2znAAn8CwnAxG5mUwanAmo/IyGTU4g9GDMxk9OJPC7FDfmtW2uRbe/j289ksXFINGuuac3DHuPn+CGw6cOfjg17bUuy/z1++EUDbM+oILnppdruZSswsa9kFmgRsEUDDR3edPdBc+NlS4z2yscI9L18Gm51wwfPQamHOV6yuJRuDFH8NLP3Nfxhf90TW1dYeWBlj2I3jj165mlJHnakqzvgif+tGBYaQK7z0A/7wBIs2u1tdS7857/k2QVdg9ZToa1TtdcOWOTvpHWUAYg7tGY0tZPZvK6ti8r54dFQ3sqGigpLKRvbVNxP9XSA/62sNi9OAwk4ZmMakom0lDs8kK9eJ5pqIRQI+sT2HvWlh6HWx71X3BDhrh+k4GDYfwEFcTKd/khgS3X4vSQSADsobACZ/bHwwdbXoeHl0EzXVw2k0QzPTCyAuk2j3uHNKyvFvYDT/OzIehH4Gi41yzWFsT0paXYMnXXLPf7C/BJ37gmvye/28XmPnHwAW/czWrml3wj6+7CzRHzYXzf+1C7sVbYfliV/6PXQtzvwpph1h9sbXR1fTqy1zNqui47rkQs2YXvPRTWPkn1/x4/GVutoHcUUf/3p2wgDDmQzRHouysbGS7FxrbyhvYXuFuW8vraWrd31cwIjeDyV5YTC7K4pgh2UwckkV6MAVNVskQaXZfsIfSVA3lxe4v74zBrmaSkbe/6evD1O6FR78MW150z30ByB4Og4a5GoD43Hu31Llbc52bsqWldv/xBZNdH82m5yBvHJz3S9f8Fm/zi/DY1S7QZl4Bqx9zc4N94nswZ9GBzWb7NsKz34f1j7vaT2a+2y8+EO++pdYFQ0vdgZ8zfCacfC1MOe/wmuLa1JfDq7fDW79zgxVmfcH1Rb39e0DhxC/DKd+EcMHhv/eHsIAw5ijEYkpJZSMb9tbywd5aNuxxt8376miNuv8/PoGx+WEmDsliXEGYMflhxuZnMqYgzLBB6fh8fai5qqfEYrDvAxcs4cIPHz4ci7kBA3tWwe5V7n7fRph6Hsz/r87/4m+ogMe/Dmv/7qZnWXina3LrzNZXXed/tNkbYRbbf0vLcl/S4QLX5BYudLWeN34NFZvd6LZ517ih0WmZboBA1fb9Ax0ayl3QtAWPz+/Kt/wPLnSOv9SNShs8zpWlaodrlnv3r66mNefLLkg15pqgNObKWDAJjj37iP4ZLCCMSYLWaIyt++q94Kjjgz21bCytZUdFY/vV4uBGV43MzWDk4ExG5mV4t0xG5KaTHw6Rn5VGVijQt/o8+hpVKNvgvkiTcR1LLOpqHq/+HHau8GpTXtMZXfiOPfZcOP07nffJlG2A538I65Yk3v+Rz8BFfziioltAGNODojFlT00T2/bVs7XcNVGVVLq+jpLKRirqWw56TVrAR0E4jfysEGPyM5lQmMV4b8TV+MLwkV8YaHqWKmx7DVb8wTVLDR7nmr/a7tuaiNqvfYm654fq74jXWOleI74ONZHggdPHHAYLCGN6kbrmCDsrG9ld7cKivK6FffXNlNe1UFbbzNZy14Eei/uvWZCVxuBwGnmZ7n6wFyYj8zIYMziTMflhhmSHrCnLHLZDBYT9WWJMD8sKBZhc5KYM6UxTa5Rt5Q1sLqujuLSO3TVNVNS1UNHQwsbSOirqW6hsaDlg5FUo4GNU+8irzAMeD8tNJ5wWwG8BYg6DBYQxvVB60P+hIdIajbGrqvGAEVfbyuvZUdHIW1sq2qcmiRcK+MhM85OZFiAjzc+wnHTGF4QZVxBmXGEW4wvCDM/NsCAxgAWEMX1W0O9jTL4bMdWRqlLV0NoeHLurG6lvjtLUGqWhpe0WYWdVI4+u3Nl+5Tm44fw5GUFyM4LkZqaRlxkkLzONvHB8M1eQweEQeZlBcjKD5GakkRboBZMYmm5lAWFMPyQi5IXdl/rxo3IPeayqsq+uhS376tmyr46dlY1UNrRS1dhKVUMLZXXNfLC3jqqGFupbop2+TzjN7wIlHGRkbiaj8/c3c40ZnMmQQSEygn4brdWHWEAYM8CJCIXZIQqzQ8wZl2AqjjhNrVEqG1pcH0h9KxUNLVQ3tFDlBUrbvuKyOp7fUEpL5MDJCIN+YVB6kEEZ7paTEaQwK9T++YXZIQqz3NDfvMw0cjODBI9mskVzVCwgjDFdlh70Mywng2E5H37FdCymlNY2t/eNlNe3UN3YSk1jq7tvilDV0ELx3lrK6prbLzrsKDsUINdr0hqaHaIoJ52hg9IZkh1iyKB0skIB0oM+0oN+dwv4yE4PWpNXN0hqQIjIAuDngB/4var+uMP+EPAnYBZQDlyiqlvj9o8G1gLfV9WfJbOsxpju5fMJRTnpFOWkf2jNRFWpbmylrLaZ0trm9lFalfWuVlLZ4IYDby2v580tFVQ3th7y/USgaFD6ARMxjsrLJC8c3F+DSQ8yKCNgzV6HkLSAEBE/cCdwJlACvC0iS1R1bdxhVwKVqjpRRC4FbgXiJ4u/DXgyWWU0xvQOIkJuZhq5mWkcM7TzkVttmlqjlNY0s7e2ifrmCE2tMZojrhO+qTVGRX0LOyobKKlo5NXifQdNxhjP7xOyQgGy0wNkhQIMSg8SDvlJC/gI+n2kBXykefd5mWmMyMtgZG4Gw3MzGJabnppp43tIMmsQc4BiVd0MICIPAAtxNYI2C4Hve48fBn4lIqKqKiLnA1uA+iSW0RjTB6UH/YzOdx3hXdEcibK7qokqr4mrpqmVmsYI1Y2t1DW3UtsUoa4pQm2zu99X10JrNEZLJEaLd98ciVHT1HpQ0AwOp3lDh/1kBP1kePe5cRc1tl/cGE6jICtEQXaIcFrvr7kkMyBGADvinpcAJ3V2jKpGRKQayBeRJuAGXO3jus4+QEQWAYsARo9O/rzpxpi+KRTwM7bg6Bcoao5E2VPdxM6qRnZWNrKrqom9tU00tQ0dbo3S1BKltLa5/YLGhk5GfqUHfS4sskLkh9O8QAm2B0tbTSYrFCAzzdVuMkN+wmmuz6UnwqW3dlJ/H7hdVesO9UNQ1buBu8FNtdEzRTPGDFShgL/Ta08609QapaLeje4qr29hX20z++qaKfPu99W1sKemiXW7a6hsaKWxtfOhxG1EIJwWaK+5fGLKUL5z7tSjObWEkhkQO4H4VS5GetsSHVMiIgEgB9dZfRJwoYj8BMgFYiLSpKq/SmJ5jTGm26UH/Qz3+iy6oi1Qapsi1DVHaGiJUN8cob45Sn1LxNVUmiPUexc7NrREGdbF9z5cyQyIt4FjRGQcLgguBS7rcMwS4PPA68CFwPPqZg9sX/VDRL4P1Fk4GGMGgrZA6Q2SFhBen8I1wFO4Ya6LVXWNiNwCLFfVJcA9wJ9FpBiowIWIMcaYXsCm+zbGmAHsUNN926WGxhhjErKAMMYYk5AFhDHGmIQsIIwxxiRkAWGMMSYhCwhjjDEJ9ZthriJSBmw7ircoAPZ1U3F6Ezuvvqe/npudV+80RlULE+3oNwFxtERkeWdjgfsyO6++p7+em51X32NNTMYYYxKygDDGGJOQBcR+d6e6AEli59X39Ndzs/PqY6wPwhhjTEJWgzDGGJOQBYQxxpiEBnxAiMgCEdkgIsUicmOqy3M0RGSxiJSKyOq4bYNF5BkR2ejd56WyjEdCREaJyDIRWSsia0TkWm97nz43EUkXkbdE5D3vvH7gbR8nIm96v5MPikhaqst6JETELyLviMjj3vP+cl5bReR9EXlXRJZ72/r072JnBnRAiIgfuBM4C5gKfFZEun9h157zR2BBh203As+p6jHAc97zviYCfEtVpwJzga96/059/dyagdNV9XhgBrBAROYCt+LWZJ8IVAJXprCMR+NaYF3c8/5yXgCnqeqMuOsf+vrvYkIDOiCAOUCxqm5W1RbgAWBhist0xFT1JdzKfPEWAvd6j+8Fzu/RQnUDVd2tqiu9x7W4L50R9PFzU6fOexr0bgqcDjzsbe9z5wUgIiOBc4Dfe8+FfnBeh9Cnfxc7M9ADYgSwI+55ibetPxmqqru9x3uAoakszNESkbHATOBN+sG5ec0w7wKlwDPAJqBKVSPeIX31d/IO4D+BmPc8n/5xXuBC/GkRWSEii7xtff53MZGkrUlteh9VVRHps+OaRSQLeAT4uqrWuD9Knb56bqoaBWaISC7wGHBsiot01ETkXKBUVVeIyPxUlycJPqaqO0VkCPCMiKyP39lXfxcTGeg1iJ3AqLjnI71t/cleERkG4N2Xprg8R0REgrhw+IuqPupt7hfnBqCqVcAyYB6QKyJtf7z1xd/Jk4HzRGQrrtn2dODn9P3zAkBVd3r3pbhQn0M/+l2MN9AD4m3gGG90RRpwKbAkxWXqbkuAz3uPPw/8PYVlOSJe+/U9wDpVvS1uV58+NxEp9GoOiEgGcCauf2UZcKF3WJ87L1W9SVVHqupY3P+p51X1cvr4eQGISFhEstseA58EVtPHfxc7M+CvpBaRs3HtpX5gsar+KMVFOmIicj8wHzf98F7ge8D/Ax4CRuOmQ79YVTt2ZPdqIvIx4GXgffa3af8Xrh+iz56biByH69D04/5Ye0hVbxGR8bi/vAcD7wBXqGpz6kp65LwmputU9dz+cF7eOTzmPQ0Af1XVH4lIPn34d7EzAz4gjDHGJDbQm5iMMcZ0wgLCGGNMQhYQxhhjErKAMMYYk5AFhDHGmIQsIIzpBURkftusp8b0FhYQxhhjErKAMOYwiMgV3hoO74rIXd5ke3Uicru3psNzIlLoHTtDRN4QkVUi8ljbGgEiMlFEnvXWgVgpIhO8t88SkYdFZL2I/EXiJ5syJgUsIIzpIhGZAlwCnKyqM4AocDkQBpar6keAF3FXsAP8CbhBVY/DXQXetv0vwJ3eOhAfBdpmAZ0JfB23Nsl43JxGxqSMzeZqTNedAcwC3vb+uM/ATcoWAx70jrkPeFREcoBcVX3R234v8DdvHp8RqvoYgKo2AXjv95aqlnjP3wXGAq8k/7SMScwCwpiuE+BeVb3pgI0i3+1w3JHOXxM/L1EU+/9pUsyamIzpuueAC711ANrWIR6D+3/UNkvpZcArqloNVIrIKd72zwEveivilYjI+d57hEQks0fPwpgusr9QjOkiVV0rIt/BrSbmA1qBrwL1wBxvXymunwLctM+/9QJgM/BFb/vngLtE5BbvPS7qwdMwpstsNldjjpKI1KlqVqrLYUx3syYmY4wxCVkNwhhjTEJWgzDGGJOQBYQxxpiELCCMMcYkZAFhjDEmIQsIY4wxCf1/STHS5Y0YS8sAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cVX3m4TeyPMz"
      },
      "source": [
        "# Train CNN with spec on California data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KrrC7o_AzQET"
      },
      "source": [
        "### model spec"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rPkBS9eNyNmy",
        "outputId": "49bf7fe7-7600-443c-a93c-f7f1dfbb5376"
      },
      "source": [
        "model_cnn = model_cnn_spec(400, 64, 16)\n",
        "\n",
        "LR = 0.0001\n",
        "model_cnn.compile(\n",
        "    optimizer=keras.optimizers.Adam(learning_rate=LR),\n",
        "    loss=keras.losses.SparseCategoricalCrossentropy(),\n",
        "    metrics=[keras.metrics.SparseCategoricalAccuracy()],)\n",
        "\n",
        "model_cnn.summary() "
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_4 (InputLayer)         [(None, 400, 3)]          0         \n",
            "_________________________________________________________________\n",
            "stft_2 (STFT)                (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude_2 (Magnitude)      (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude_to_decibel_2 (Magn (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "max_abs_scaler_2 (MaxABSScal (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "conv2d (Conv2D)              (None, 22, 33, 32)        896       \n",
            "_________________________________________________________________\n",
            "batch_normalization_6 (Batch (None, 22, 33, 32)        128       \n",
            "_________________________________________________________________\n",
            "tf.nn.relu (TFOpLambda)      (None, 22, 33, 32)        0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D) (None, 11, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 11, 16, 64)        18496     \n",
            "_________________________________________________________________\n",
            "batch_normalization_7 (Batch (None, 11, 16, 64)        256       \n",
            "_________________________________________________________________\n",
            "tf.nn.relu_1 (TFOpLambda)    (None, 11, 16, 64)        0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2 (None, 5, 8, 64)          0         \n",
            "_________________________________________________________________\n",
            "conv2d_2 (Conv2D)            (None, 5, 8, 128)         73856     \n",
            "_________________________________________________________________\n",
            "batch_normalization_8 (Batch (None, 5, 8, 128)         512       \n",
            "_________________________________________________________________\n",
            "tf.nn.relu_2 (TFOpLambda)    (None, 5, 8, 128)         0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2 (None, 2, 4, 128)         0         \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dropout_14 (Dropout)         (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dense_19 (Dense)             (None, 80)                82000     \n",
            "_________________________________________________________________\n",
            "dropout_15 (Dropout)         (None, 80)                0         \n",
            "_________________________________________________________________\n",
            "dense_20 (Dense)             (None, 3)                 243       \n",
            "=================================================================\n",
            "Total params: 176,387\n",
            "Trainable params: 175,939\n",
            "Non-trainable params: 448\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1kPUkqz6oPSN"
      },
      "source": [
        "# Load weights example\n",
        "model_cnn.load_weights('WEIGHTS/weights_model_cnn_spec.hd5')"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6y9zU65F7KJ9",
        "outputId": "da100da7-c097-4e8a-ca14-b54ef189f0e8"
      },
      "source": [
        "# evaluate model example\n",
        "\n",
        "print(\"Evaluate transformer based model on SAKHALIN test data\")\n",
        "results_sakh = model_cnn.evaluate(X_test_sakh, y_test_sakh, batch_size=128)\n",
        "print(\"test loss, test acc:\", results_sakh)\n",
        "\n",
        "print(\"Evaluate transformer based model on DAGESTAN test data\")\n",
        "results_dag = model_cnn.evaluate(X_test_dag, y_test_dag, batch_size=128)\n",
        "print(\"test loss, test acc:\", results_dag)\n",
        "\n",
        "print(\"Evaluate transformer based model on CALIFORNIA test data\")\n",
        "results_cal = model_cnn.evaluate(X_test)\n",
        "print(\"test loss, test acc:\", results_cal)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 4s 4ms/step - loss: 0.2592 - sparse_categorical_accuracy: 0.9145\n",
            "test loss, test acc: [0.2591601014137268, 0.9144824743270874]\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4864 - sparse_categorical_accuracy: 0.8612\n",
            "test loss, test acc: [0.48637187480926514, 0.8611929416656494]\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 13s 6ms/step - loss: 0.0407 - sparse_categorical_accuracy: 0.9882\n",
            "test loss, test acc: [0.04070014879107475, 0.9881877899169922]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a7Or4Dv9zp0p"
      },
      "source": [
        "### Train the model with different random seeds"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A7PkGcPP9q5U",
        "outputId": "a2bb063e-87e7-4778-cbbc-8712bc69e227"
      },
      "source": [
        "%%time\n",
        "\n",
        "interations = 7\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 5, restore_best_weights = True, verbose = 1),\n",
        "    keras.callbacks.ModelCheckpoint('w_cnn_epoch_hpa{epoch}.h5', save_weights_only=True),\n",
        "]\n",
        "\n",
        "sparse_categorical_accuracy = []\n",
        "loss = []\n",
        "val_loss = []\n",
        "val_sparse_categorical_accuracy = []\n",
        "\n",
        "cal_top_1 = []\n",
        "sakh_top_1 = []\n",
        "dag_top_1 = []\n",
        "\n",
        "for i in range(interations):\n",
        "  print(\"Load dataset with random state {}\".format(i)) \n",
        "  X_train, X_test = h5_tts('/content/scsn_ps_2000_2017_shuf.hdf5',\\\n",
        "                             batch_size = 480,\\\n",
        "                             test_size = 0.2, random_state = i, shuffle = False)\n",
        "\n",
        "  print(\"compile the model\")\n",
        "  model_cnn = model_cnn_spec(400, 64, 16)\n",
        "\n",
        "  LR = 0.0001\n",
        "  model_cnn.compile(\n",
        "      optimizer=keras.optimizers.Adam(learning_rate=LR),\n",
        "      loss=keras.losses.SparseCategoricalCrossentropy(),\n",
        "      metrics=[keras.metrics.SparseCategoricalAccuracy()],)\n",
        "\n",
        "  print(\"Fit model on training data step {}\".format(i+1))\n",
        "  history = model_cnn.fit(\n",
        "    X_train,\n",
        "    #shuffle=False,\n",
        "    epochs=400,\n",
        "    validation_data=(X_test),\n",
        "    callbacks = callbacks\n",
        "    )\n",
        "  \n",
        "  #save convergance curve \n",
        "  sparse_categorical_accuracy.append(history.history['sparse_categorical_accuracy'])\n",
        "  loss.append(history.history['loss'])\n",
        "  val_sparse_categorical_accuracy.append(history.history['val_sparse_categorical_accuracy'])\n",
        "  val_loss.append(history.history['val_loss'])\n",
        "\n",
        "  print(\"Evaluate transformer based model on SAKHALIN test data\")\n",
        "  sakh_top_1.append(model_cnn.evaluate(X_test_sakh, y_test_sakh, batch_size=128)[1])\n",
        "\n",
        "  print(\"Evaluate transformer based model on DAGESTAN test data\")\n",
        "  dag_top_1.append(model_cnn.evaluate(X_test_dag, y_test_dag, batch_size=128)[1])\n",
        "  \n",
        "  print(\"Evaluate transformer based model on CALIFORNIA test data\")\n",
        "  cal_top_1.append(model_cnn.evaluate(X_test)[1])\n",
        "\n",
        "  model_cnn.save_weights('/content/drive/MyDrive/MODELS/w_model_cnn_seeds.hd5', save_format='h5')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Load dataset with random state 0\n",
            "compile the model\n",
            "Fit model on training data step 1\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 347s 42ms/step - loss: 0.1101 - sparse_categorical_accuracy: 0.9664 - val_loss: 0.0623 - val_sparse_categorical_accuracy: 0.9818\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 317s 40ms/step - loss: 0.0655 - sparse_categorical_accuracy: 0.9816 - val_loss: 0.0540 - val_sparse_categorical_accuracy: 0.9842\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0596 - sparse_categorical_accuracy: 0.9833 - val_loss: 0.0506 - val_sparse_categorical_accuracy: 0.9851\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 287s 36ms/step - loss: 0.0564 - sparse_categorical_accuracy: 0.9842 - val_loss: 0.0489 - val_sparse_categorical_accuracy: 0.9857\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 286s 36ms/step - loss: 0.0546 - sparse_categorical_accuracy: 0.9848 - val_loss: 0.0469 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 288s 36ms/step - loss: 0.0532 - sparse_categorical_accuracy: 0.9852 - val_loss: 0.0470 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0520 - sparse_categorical_accuracy: 0.9855 - val_loss: 0.0474 - val_sparse_categorical_accuracy: 0.9861\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0509 - sparse_categorical_accuracy: 0.9858 - val_loss: 0.0450 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0502 - sparse_categorical_accuracy: 0.9859 - val_loss: 0.0454 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0496 - sparse_categorical_accuracy: 0.9861 - val_loss: 0.0437 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0489 - sparse_categorical_accuracy: 0.9863 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0482 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0478 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9875\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0474 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9874\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0470 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0432 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0466 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0438 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0462 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0424 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0459 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0420 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0456 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0421 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0454 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9874\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0450 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0419 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0448 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0422 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0447 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0411 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0443 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0421 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0441 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0420 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0438 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0413 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0412 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0434 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0418 - val_sparse_categorical_accuracy: 0.9879\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00028: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 5ms/step - loss: 0.2667 - sparse_categorical_accuracy: 0.9151\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 3ms/step - loss: 0.4796 - sparse_categorical_accuracy: 0.8546\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 10s 5ms/step - loss: 0.0411 - sparse_categorical_accuracy: 0.9880\n",
            "Load dataset with random state 1\n",
            "compile the model\n",
            "Fit model on training data step 2\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 322s 40ms/step - loss: 0.1052 - sparse_categorical_accuracy: 0.9682 - val_loss: 0.0600 - val_sparse_categorical_accuracy: 0.9824\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 289s 36ms/step - loss: 0.0655 - sparse_categorical_accuracy: 0.9815 - val_loss: 0.0561 - val_sparse_categorical_accuracy: 0.9832\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 290s 36ms/step - loss: 0.0595 - sparse_categorical_accuracy: 0.9833 - val_loss: 0.0497 - val_sparse_categorical_accuracy: 0.9854\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 291s 37ms/step - loss: 0.0565 - sparse_categorical_accuracy: 0.9842 - val_loss: 0.0488 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0544 - sparse_categorical_accuracy: 0.9847 - val_loss: 0.0476 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0531 - sparse_categorical_accuracy: 0.9851 - val_loss: 0.0478 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0520 - sparse_categorical_accuracy: 0.9855 - val_loss: 0.0458 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0510 - sparse_categorical_accuracy: 0.9857 - val_loss: 0.0460 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0500 - sparse_categorical_accuracy: 0.9860 - val_loss: 0.0452 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0494 - sparse_categorical_accuracy: 0.9861 - val_loss: 0.0435 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0487 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0445 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0483 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0449 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0477 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0433 - val_sparse_categorical_accuracy: 0.9875\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0473 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0427 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0470 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0466 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0424 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0461 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0433 - val_sparse_categorical_accuracy: 0.9875\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0458 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0448 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 299s 38ms/step - loss: 0.0456 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0418 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 298s 38ms/step - loss: 0.0453 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0432 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0450 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0421 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0446 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0421 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 295s 37ms/step - loss: 0.0445 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0418 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0442 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0417 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 298s 38ms/step - loss: 0.0441 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0427 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0423 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0415 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0434 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0414 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 29/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0433 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0421 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 30/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0431 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0407 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 31/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0428 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0408 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 32/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0428 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0417 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 33/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0426 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0407 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 34/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0424 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0408 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 35/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0422 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9873\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00035: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 4ms/step - loss: 0.2639 - sparse_categorical_accuracy: 0.9126\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.5176 - sparse_categorical_accuracy: 0.8513\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 10s 5ms/step - loss: 0.0407 - sparse_categorical_accuracy: 0.9882\n",
            "Load dataset with random state 2\n",
            "compile the model\n",
            "Fit model on training data step 3\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 327s 41ms/step - loss: 0.1080 - sparse_categorical_accuracy: 0.9670 - val_loss: 0.0603 - val_sparse_categorical_accuracy: 0.9821\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 293s 37ms/step - loss: 0.0654 - sparse_categorical_accuracy: 0.9816 - val_loss: 0.0554 - val_sparse_categorical_accuracy: 0.9839\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 292s 37ms/step - loss: 0.0595 - sparse_categorical_accuracy: 0.9833 - val_loss: 0.0502 - val_sparse_categorical_accuracy: 0.9853\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 294s 37ms/step - loss: 0.0566 - sparse_categorical_accuracy: 0.9842 - val_loss: 0.0481 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 296s 37ms/step - loss: 0.0545 - sparse_categorical_accuracy: 0.9848 - val_loss: 0.0483 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0531 - sparse_categorical_accuracy: 0.9852 - val_loss: 0.0460 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0519 - sparse_categorical_accuracy: 0.9855 - val_loss: 0.0450 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0510 - sparse_categorical_accuracy: 0.9858 - val_loss: 0.0458 - val_sparse_categorical_accuracy: 0.9866\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0501 - sparse_categorical_accuracy: 0.9861 - val_loss: 0.0445 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 299s 38ms/step - loss: 0.0493 - sparse_categorical_accuracy: 0.9862 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 299s 38ms/step - loss: 0.0486 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0437 - val_sparse_categorical_accuracy: 0.9874\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 299s 38ms/step - loss: 0.0482 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0458 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 299s 38ms/step - loss: 0.0476 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0471 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0430 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0469 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0434 - val_sparse_categorical_accuracy: 0.9875\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0464 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0434 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0460 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0425 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0457 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0431 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0454 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0426 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0450 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 299s 38ms/step - loss: 0.0450 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0415 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0447 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0414 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 299s 38ms/step - loss: 0.0444 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0415 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0442 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0423 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0441 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0411 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 299s 38ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0419 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0435 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0416 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0434 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0423 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 29/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0432 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0422 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 30/400\n",
            "7957/7957 [==============================] - 298s 37ms/step - loss: 0.0431 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0409 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 31/400\n",
            "7957/7957 [==============================] - 298s 38ms/step - loss: 0.0429 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0414 - val_sparse_categorical_accuracy: 0.9883\n",
            "Epoch 32/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0426 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0423 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 33/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0425 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0407 - val_sparse_categorical_accuracy: 0.9883\n",
            "Epoch 34/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0422 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0402 - val_sparse_categorical_accuracy: 0.9883\n",
            "Epoch 35/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0421 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0410 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 36/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0421 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0424 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 37/400\n",
            "7957/7957 [==============================] - 299s 38ms/step - loss: 0.0419 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0408 - val_sparse_categorical_accuracy: 0.9883\n",
            "Epoch 38/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0416 - sparse_categorical_accuracy: 0.9883 - val_loss: 0.0413 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 39/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0416 - sparse_categorical_accuracy: 0.9883 - val_loss: 0.0412 - val_sparse_categorical_accuracy: 0.9882\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00039: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 4ms/step - loss: 0.2610 - sparse_categorical_accuracy: 0.9159\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 3ms/step - loss: 0.4604 - sparse_categorical_accuracy: 0.8603\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 10s 5ms/step - loss: 0.0402 - sparse_categorical_accuracy: 0.9883\n",
            "Load dataset with random state 3\n",
            "compile the model\n",
            "Fit model on training data step 4\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 328s 41ms/step - loss: 0.1072 - sparse_categorical_accuracy: 0.9673 - val_loss: 0.0593 - val_sparse_categorical_accuracy: 0.9826\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 297s 37ms/step - loss: 0.0654 - sparse_categorical_accuracy: 0.9816 - val_loss: 0.0545 - val_sparse_categorical_accuracy: 0.9842\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0594 - sparse_categorical_accuracy: 0.9834 - val_loss: 0.0502 - val_sparse_categorical_accuracy: 0.9853\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0563 - sparse_categorical_accuracy: 0.9842 - val_loss: 0.0481 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0543 - sparse_categorical_accuracy: 0.9848 - val_loss: 0.0476 - val_sparse_categorical_accuracy: 0.9862\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0528 - sparse_categorical_accuracy: 0.9852 - val_loss: 0.0464 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 303s 38ms/step - loss: 0.0516 - sparse_categorical_accuracy: 0.9855 - val_loss: 0.0453 - val_sparse_categorical_accuracy: 0.9868\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 303s 38ms/step - loss: 0.0507 - sparse_categorical_accuracy: 0.9858 - val_loss: 0.0453 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 304s 38ms/step - loss: 0.0499 - sparse_categorical_accuracy: 0.9860 - val_loss: 0.0464 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0493 - sparse_categorical_accuracy: 0.9862 - val_loss: 0.0494 - val_sparse_categorical_accuracy: 0.9855\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0487 - sparse_categorical_accuracy: 0.9863 - val_loss: 0.0437 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 302s 38ms/step - loss: 0.0481 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0433 - val_sparse_categorical_accuracy: 0.9875\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0477 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 302s 38ms/step - loss: 0.0472 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0429 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 302s 38ms/step - loss: 0.0468 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0433 - val_sparse_categorical_accuracy: 0.9875\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 303s 38ms/step - loss: 0.0464 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0428 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 306s 38ms/step - loss: 0.0461 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0427 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 305s 38ms/step - loss: 0.0455 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0425 - val_sparse_categorical_accuracy: 0.9875\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 304s 38ms/step - loss: 0.0454 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0423 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 303s 38ms/step - loss: 0.0451 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0419 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 302s 38ms/step - loss: 0.0449 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0415 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 304s 38ms/step - loss: 0.0446 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0441 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 305s 38ms/step - loss: 0.0444 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0425 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 304s 38ms/step - loss: 0.0441 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0416 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 304s 38ms/step - loss: 0.0439 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0422 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 304s 38ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0416 - val_sparse_categorical_accuracy: 0.9880\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00026: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 4ms/step - loss: 0.2649 - sparse_categorical_accuracy: 0.9136\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.5020 - sparse_categorical_accuracy: 0.8516\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 11s 5ms/step - loss: 0.0415 - sparse_categorical_accuracy: 0.9879\n",
            "Load dataset with random state 4\n",
            "compile the model\n",
            "Fit model on training data step 5\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 331s 41ms/step - loss: 0.1044 - sparse_categorical_accuracy: 0.9684 - val_loss: 0.0606 - val_sparse_categorical_accuracy: 0.9822\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0652 - sparse_categorical_accuracy: 0.9818 - val_loss: 0.0528 - val_sparse_categorical_accuracy: 0.9845\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 303s 38ms/step - loss: 0.0594 - sparse_categorical_accuracy: 0.9834 - val_loss: 0.0503 - val_sparse_categorical_accuracy: 0.9852\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 305s 38ms/step - loss: 0.0563 - sparse_categorical_accuracy: 0.9843 - val_loss: 0.0477 - val_sparse_categorical_accuracy: 0.9861\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 304s 38ms/step - loss: 0.0542 - sparse_categorical_accuracy: 0.9848 - val_loss: 0.0479 - val_sparse_categorical_accuracy: 0.9860\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 306s 38ms/step - loss: 0.0528 - sparse_categorical_accuracy: 0.9853 - val_loss: 0.0458 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 304s 38ms/step - loss: 0.0517 - sparse_categorical_accuracy: 0.9856 - val_loss: 0.0463 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 308s 39ms/step - loss: 0.0508 - sparse_categorical_accuracy: 0.9858 - val_loss: 0.0454 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 306s 38ms/step - loss: 0.0499 - sparse_categorical_accuracy: 0.9861 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9871\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 306s 38ms/step - loss: 0.0493 - sparse_categorical_accuracy: 0.9862 - val_loss: 0.0452 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 306s 39ms/step - loss: 0.0487 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9874\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 306s 38ms/step - loss: 0.0481 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9874\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 306s 38ms/step - loss: 0.0476 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0453 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 306s 38ms/step - loss: 0.0472 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0467 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0443 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 307s 39ms/step - loss: 0.0464 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9873\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00016: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 5ms/step - loss: 0.2590 - sparse_categorical_accuracy: 0.9095\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4911 - sparse_categorical_accuracy: 0.8462\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 12s 6ms/step - loss: 0.0436 - sparse_categorical_accuracy: 0.9874\n",
            "Load dataset with random state 5\n",
            "compile the model\n",
            "Fit model on training data step 6\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 333s 42ms/step - loss: 0.1085 - sparse_categorical_accuracy: 0.9671 - val_loss: 0.0613 - val_sparse_categorical_accuracy: 0.9819\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 300s 38ms/step - loss: 0.0653 - sparse_categorical_accuracy: 0.9817 - val_loss: 0.0535 - val_sparse_categorical_accuracy: 0.9842\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 301s 38ms/step - loss: 0.0597 - sparse_categorical_accuracy: 0.9833 - val_loss: 0.0521 - val_sparse_categorical_accuracy: 0.9849\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 306s 38ms/step - loss: 0.0565 - sparse_categorical_accuracy: 0.9842 - val_loss: 0.0482 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 305s 38ms/step - loss: 0.0546 - sparse_categorical_accuracy: 0.9848 - val_loss: 0.0503 - val_sparse_categorical_accuracy: 0.9851\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 306s 38ms/step - loss: 0.0530 - sparse_categorical_accuracy: 0.9852 - val_loss: 0.0472 - val_sparse_categorical_accuracy: 0.9863\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 306s 39ms/step - loss: 0.0519 - sparse_categorical_accuracy: 0.9855 - val_loss: 0.0484 - val_sparse_categorical_accuracy: 0.9857\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0509 - sparse_categorical_accuracy: 0.9858 - val_loss: 0.0462 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 307s 39ms/step - loss: 0.0500 - sparse_categorical_accuracy: 0.9860 - val_loss: 0.0444 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 306s 39ms/step - loss: 0.0494 - sparse_categorical_accuracy: 0.9863 - val_loss: 0.0452 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 307s 39ms/step - loss: 0.0489 - sparse_categorical_accuracy: 0.9863 - val_loss: 0.0442 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0482 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0446 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 308s 39ms/step - loss: 0.0478 - sparse_categorical_accuracy: 0.9866 - val_loss: 0.0434 - val_sparse_categorical_accuracy: 0.9874\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 310s 39ms/step - loss: 0.0474 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0435 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0471 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0433 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0466 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0440 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 308s 39ms/step - loss: 0.0463 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0424 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 307s 39ms/step - loss: 0.0459 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0427 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 308s 39ms/step - loss: 0.0457 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0426 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 307s 39ms/step - loss: 0.0453 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0422 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0451 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0432 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 312s 39ms/step - loss: 0.0448 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0418 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 311s 39ms/step - loss: 0.0445 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0417 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0443 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0417 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 307s 39ms/step - loss: 0.0441 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0416 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0438 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0417 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 310s 39ms/step - loss: 0.0437 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0415 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0434 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0416 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 29/400\n",
            "7957/7957 [==============================] - 310s 39ms/step - loss: 0.0433 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0413 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 30/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0431 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0411 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 31/400\n",
            "7957/7957 [==============================] - 310s 39ms/step - loss: 0.0430 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0411 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 32/400\n",
            "7957/7957 [==============================] - 308s 39ms/step - loss: 0.0426 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0410 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 33/400\n",
            "7957/7957 [==============================] - 308s 39ms/step - loss: 0.0427 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0411 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 34/400\n",
            "7957/7957 [==============================] - 308s 39ms/step - loss: 0.0424 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0414 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 35/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0424 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0418 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 36/400\n",
            "7957/7957 [==============================] - 310s 39ms/step - loss: 0.0422 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0406 - val_sparse_categorical_accuracy: 0.9883\n",
            "Epoch 37/400\n",
            "7957/7957 [==============================] - 310s 39ms/step - loss: 0.0420 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.0420 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 38/400\n",
            "7957/7957 [==============================] - 310s 39ms/step - loss: 0.0419 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0411 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 39/400\n",
            "7957/7957 [==============================] - 310s 39ms/step - loss: 0.0418 - sparse_categorical_accuracy: 0.9882 - val_loss: 0.0408 - val_sparse_categorical_accuracy: 0.9883\n",
            "Epoch 40/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0416 - sparse_categorical_accuracy: 0.9883 - val_loss: 0.0424 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 41/400\n",
            "7957/7957 [==============================] - 312s 39ms/step - loss: 0.0415 - sparse_categorical_accuracy: 0.9883 - val_loss: 0.0413 - val_sparse_categorical_accuracy: 0.9882\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00041: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 4ms/step - loss: 0.2643 - sparse_categorical_accuracy: 0.9163\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.5268 - sparse_categorical_accuracy: 0.8542\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 11s 6ms/step - loss: 0.0406 - sparse_categorical_accuracy: 0.9883\n",
            "Load dataset with random state 6\n",
            "compile the model\n",
            "Fit model on training data step 7\n",
            "Epoch 1/400\n",
            "7957/7957 [==============================] - 337s 42ms/step - loss: 0.1085 - sparse_categorical_accuracy: 0.9671 - val_loss: 0.0623 - val_sparse_categorical_accuracy: 0.9815\n",
            "Epoch 2/400\n",
            "7957/7957 [==============================] - 307s 39ms/step - loss: 0.0654 - sparse_categorical_accuracy: 0.9816 - val_loss: 0.0534 - val_sparse_categorical_accuracy: 0.9841\n",
            "Epoch 3/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0596 - sparse_categorical_accuracy: 0.9833 - val_loss: 0.0509 - val_sparse_categorical_accuracy: 0.9853\n",
            "Epoch 4/400\n",
            "7957/7957 [==============================] - 311s 39ms/step - loss: 0.0565 - sparse_categorical_accuracy: 0.9842 - val_loss: 0.0490 - val_sparse_categorical_accuracy: 0.9856\n",
            "Epoch 5/400\n",
            "7957/7957 [==============================] - 311s 39ms/step - loss: 0.0545 - sparse_categorical_accuracy: 0.9847 - val_loss: 0.0477 - val_sparse_categorical_accuracy: 0.9859\n",
            "Epoch 6/400\n",
            "7957/7957 [==============================] - 312s 39ms/step - loss: 0.0531 - sparse_categorical_accuracy: 0.9851 - val_loss: 0.0464 - val_sparse_categorical_accuracy: 0.9865\n",
            "Epoch 7/400\n",
            "7957/7957 [==============================] - 313s 39ms/step - loss: 0.0518 - sparse_categorical_accuracy: 0.9854 - val_loss: 0.0455 - val_sparse_categorical_accuracy: 0.9867\n",
            "Epoch 8/400\n",
            "7957/7957 [==============================] - 310s 39ms/step - loss: 0.0508 - sparse_categorical_accuracy: 0.9858 - val_loss: 0.0450 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 9/400\n",
            "7957/7957 [==============================] - 311s 39ms/step - loss: 0.0501 - sparse_categorical_accuracy: 0.9860 - val_loss: 0.0449 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 10/400\n",
            "7957/7957 [==============================] - 309s 39ms/step - loss: 0.0494 - sparse_categorical_accuracy: 0.9861 - val_loss: 0.0447 - val_sparse_categorical_accuracy: 0.9870\n",
            "Epoch 11/400\n",
            "7957/7957 [==============================] - 312s 39ms/step - loss: 0.0488 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 12/400\n",
            "7957/7957 [==============================] - 311s 39ms/step - loss: 0.0482 - sparse_categorical_accuracy: 0.9865 - val_loss: 0.0432 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 13/400\n",
            "7957/7957 [==============================] - 315s 40ms/step - loss: 0.0478 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.0433 - val_sparse_categorical_accuracy: 0.9874\n",
            "Epoch 14/400\n",
            "7957/7957 [==============================] - 313s 39ms/step - loss: 0.0474 - sparse_categorical_accuracy: 0.9868 - val_loss: 0.0444 - val_sparse_categorical_accuracy: 0.9869\n",
            "Epoch 15/400\n",
            "7957/7957 [==============================] - 313s 39ms/step - loss: 0.0468 - sparse_categorical_accuracy: 0.9869 - val_loss: 0.0439 - val_sparse_categorical_accuracy: 0.9873\n",
            "Epoch 16/400\n",
            "7957/7957 [==============================] - 312s 39ms/step - loss: 0.0463 - sparse_categorical_accuracy: 0.9870 - val_loss: 0.0428 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 17/400\n",
            "7957/7957 [==============================] - 311s 39ms/step - loss: 0.0460 - sparse_categorical_accuracy: 0.9871 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9872\n",
            "Epoch 18/400\n",
            "7957/7957 [==============================] - 315s 40ms/step - loss: 0.0457 - sparse_categorical_accuracy: 0.9872 - val_loss: 0.0418 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 19/400\n",
            "7957/7957 [==============================] - 314s 39ms/step - loss: 0.0455 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0436 - val_sparse_categorical_accuracy: 0.9874\n",
            "Epoch 20/400\n",
            "7957/7957 [==============================] - 313s 39ms/step - loss: 0.0452 - sparse_categorical_accuracy: 0.9873 - val_loss: 0.0428 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 21/400\n",
            "7957/7957 [==============================] - 314s 39ms/step - loss: 0.0450 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0427 - val_sparse_categorical_accuracy: 0.9876\n",
            "Epoch 22/400\n",
            "7957/7957 [==============================] - 314s 39ms/step - loss: 0.0446 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0426 - val_sparse_categorical_accuracy: 0.9877\n",
            "Epoch 23/400\n",
            "7957/7957 [==============================] - 314s 39ms/step - loss: 0.0445 - sparse_categorical_accuracy: 0.9876 - val_loss: 0.0412 - val_sparse_categorical_accuracy: 0.9880\n",
            "Epoch 24/400\n",
            "7957/7957 [==============================] - 313s 39ms/step - loss: 0.0442 - sparse_categorical_accuracy: 0.9875 - val_loss: 0.0417 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 25/400\n",
            "7957/7957 [==============================] - 315s 40ms/step - loss: 0.0439 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0418 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 26/400\n",
            "7957/7957 [==============================] - 315s 40ms/step - loss: 0.0439 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0421 - val_sparse_categorical_accuracy: 0.9879\n",
            "Epoch 27/400\n",
            "7957/7957 [==============================] - 314s 39ms/step - loss: 0.0436 - sparse_categorical_accuracy: 0.9877 - val_loss: 0.0411 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 28/400\n",
            "7957/7957 [==============================] - 315s 40ms/step - loss: 0.0435 - sparse_categorical_accuracy: 0.9878 - val_loss: 0.0410 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 29/400\n",
            "7957/7957 [==============================] - 315s 40ms/step - loss: 0.0432 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0424 - val_sparse_categorical_accuracy: 0.9878\n",
            "Epoch 30/400\n",
            "7957/7957 [==============================] - 314s 39ms/step - loss: 0.0430 - sparse_categorical_accuracy: 0.9879 - val_loss: 0.0419 - val_sparse_categorical_accuracy: 0.9881\n",
            "Epoch 31/400\n",
            "7957/7957 [==============================] - 314s 39ms/step - loss: 0.0428 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0415 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 32/400\n",
            "7957/7957 [==============================] - 315s 40ms/step - loss: 0.0428 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0417 - val_sparse_categorical_accuracy: 0.9882\n",
            "Epoch 33/400\n",
            "7957/7957 [==============================] - 314s 39ms/step - loss: 0.0425 - sparse_categorical_accuracy: 0.9880 - val_loss: 0.0411 - val_sparse_categorical_accuracy: 0.9883\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00033: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "77/77 [==============================] - 1s 4ms/step - loss: 0.2552 - sparse_categorical_accuracy: 0.9167\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.5192 - sparse_categorical_accuracy: 0.8540\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "1990/1990 [==============================] - 11s 6ms/step - loss: 0.0410 - sparse_categorical_accuracy: 0.9881\n",
            "CPU times: user 7h 14min 49s, sys: 1h 21min 4s, total: 8h 35min 54s\n",
            "Wall time: 18h 24min 39s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qPk8M2_4zxo7"
      },
      "source": [
        "### Print collected values and scores"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "frz4yo-zz3Z5",
        "outputId": "6f94204b-a612-44f3-af39-8d095da059a6"
      },
      "source": [
        "print(val_sparse_categorical_accuracy)\n",
        "print(val_loss)\n",
        "print(sakh_top_1)\n",
        "print(dag_top_1)\n",
        "print(cal_top_1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.981822669506073, 0.9842022657394409, 0.9851214289665222, 0.9857307076454163, 0.9863117933273315, 0.9863285422325134, 0.986090898513794, 0.9868907332420349, 0.9869011640548706, 0.9873419404029846, 0.9872435331344604, 0.9872968792915344, 0.9874665141105652, 0.9873628616333008, 0.9873346090316772, 0.9872413873672485, 0.9877512454986572, 0.9877753257751465, 0.9877617359161377, 0.9873816967010498, 0.9879627227783203, 0.9878852367401123, 0.9880056381225586, 0.9876737594604492, 0.9877763986587524, 0.9881333708763123, 0.9879627227783203, 0.9879250526428223], [0.9823565483093262, 0.9831815361976624, 0.9853957295417786, 0.9858531951904297, 0.9860113263130188, 0.9859296679496765, 0.9866792559623718, 0.98662269115448, 0.9868090748786926, 0.9873167872428894, 0.9872477054595947, 0.986869752407074, 0.987517774105072, 0.9876371622085571, 0.9875900149345398, 0.9876643419265747, 0.9875251054763794, 0.9872435331344604, 0.9878444075584412, 0.9875534176826477, 0.9876968264579773, 0.9878129959106445, 0.987939715385437, 0.9878318905830383, 0.9875910878181458, 0.987938642501831, 0.9879815578460693, 0.9879962205886841, 0.9878402352333069, 0.9881846904754639, 0.988128125667572, 0.9879292249679565, 0.9882276058197021, 0.9881427884101868, 0.9872560501098633], [0.9821450710296631, 0.9839237928390503, 0.9853381514549255, 0.98601233959198, 0.9858584403991699, 0.9865902662277222, 0.9869011640548706, 0.9866258502006531, 0.9871073961257935, 0.9872288703918457, 0.98738694190979, 0.9869796633720398, 0.9872152209281921, 0.9877030849456787, 0.9875470995903015, 0.9876183271408081, 0.9877250790596008, 0.9876570105552673, 0.9876424074172974, 0.987101137638092, 0.98805171251297, 0.9880370497703552, 0.988109290599823, 0.9879418015480042, 0.9881637096405029, 0.9880087971687317, 0.9880977869033813, 0.9878978133201599, 0.9878705739974976, 0.9882422685623169, 0.9882810115814209, 0.9876716732978821, 0.9883343577384949, 0.988264262676239, 0.9881888628005981, 0.9880203008651733, 0.9883228540420532, 0.988185703754425, 0.988225519657135], [0.9826036691665649, 0.9841792583465576, 0.985348641872406, 0.9858856797218323, 0.9861976504325867, 0.9863243103027344, 0.9867619276046753, 0.9868530035018921, 0.9863285422325134, 0.9854501485824585, 0.9872947931289673, 0.9875314235687256, 0.9872424602508545, 0.9876172542572021, 0.9875199198722839, 0.9876036643981934, 0.9875785112380981, 0.9875418543815613, 0.9877250790596008, 0.9878517389297485, 0.9879459738731384, 0.9872403740882874, 0.987727165222168, 0.987974226474762, 0.9880558848381042, 0.987958550453186], [0.9821524024009705, 0.9844754934310913, 0.9852145910263062, 0.9860992431640625, 0.9859715104103088, 0.9866656064987183, 0.9866593480110168, 0.9868980050086975, 0.9870644807815552, 0.986869752407074, 0.9873597025871277, 0.9874120354652405, 0.9870121479034424, 0.9872183799743652, 0.9873115420341492, 0.9872686266899109], [0.9819158315658569, 0.9842211008071899, 0.984907865524292, 0.9859097599983215, 0.9850785136222839, 0.9862709641456604, 0.985672116279602, 0.9864583611488342, 0.9870089888572693, 0.9869650602340698, 0.9873220324516296, 0.9869168996810913, 0.9873995184898376, 0.9872853755950928, 0.9873314499855042, 0.9872947931289673, 0.9876863360404968, 0.9876486659049988, 0.987707257270813, 0.9877386689186096, 0.9876528382301331, 0.9878758192062378, 0.9879648089408875, 0.9879187345504761, 0.9877920746803284, 0.9880192875862122, 0.9880307912826538, 0.9878946542739868, 0.9881836175918579, 0.9880454540252686, 0.9881637096405029, 0.988089382648468, 0.9881051182746887, 0.9881532788276672, 0.9880014657974243, 0.9882516860961914, 0.988070547580719, 0.9882401823997498, 0.9883459210395813, 0.9876622557640076, 0.9881731867790222], [0.9814667105674744, 0.9841457009315491, 0.9852952361106873, 0.9855611324310303, 0.9858825206756592, 0.9864845275878906, 0.9867295026779175, 0.986927330493927, 0.9868812561035156, 0.9869775772094727, 0.9872403740882874, 0.9873408675193787, 0.9873879551887512, 0.986892819404602, 0.9873325228691101, 0.9876224994659424, 0.9872204661369324, 0.9878444075584412, 0.987422525882721, 0.9877418279647827, 0.9876046776771545, 0.9877145886421204, 0.9880077242851257, 0.9879470467567444, 0.9880799651145935, 0.9879438877105713, 0.9881030321121216, 0.9880810379981995, 0.9877512454986572, 0.9881166219711304, 0.9881658554077148, 0.9882003664970398, 0.9883260130882263]]\n",
            "[[0.06226234883069992, 0.05400916561484337, 0.05057425796985626, 0.04886332154273987, 0.04691261053085327, 0.047003548592329025, 0.04740159586071968, 0.04503067582845688, 0.04536796361207962, 0.043665021657943726, 0.04393979534506798, 0.04394150897860527, 0.04362957179546356, 0.04393753409385681, 0.04318135604262352, 0.04381294548511505, 0.04244311526417732, 0.04201927036046982, 0.042119428515434265, 0.04355846345424652, 0.0418761782348156, 0.04216598719358444, 0.04114806279540062, 0.04213671386241913, 0.04197687283158302, 0.04126518964767456, 0.041249509900808334, 0.04179428517818451], [0.059954605996608734, 0.05609114095568657, 0.049702297896146774, 0.04881896823644638, 0.04761482775211334, 0.04782119393348694, 0.045844342559576035, 0.04598984494805336, 0.0452064611017704, 0.04351666942238808, 0.044454723596572876, 0.04494220018386841, 0.04332348331809044, 0.04271220788359642, 0.04355284571647644, 0.04243772104382515, 0.043253134936094284, 0.044793304055929184, 0.04183521121740341, 0.04318667948246002, 0.04206375405192375, 0.04210038483142853, 0.04177296906709671, 0.0416763536632061, 0.04272482916712761, 0.04225881025195122, 0.041521020233631134, 0.04136587306857109, 0.04211023077368736, 0.04068915545940399, 0.04081328958272934, 0.04170216992497444, 0.04072049260139465, 0.040784165263175964, 0.04364384338259697], [0.0602509044110775, 0.05535998195409775, 0.050191983580589294, 0.0481000654399395, 0.048275306820869446, 0.04604862630367279, 0.04500507190823555, 0.045834120362997055, 0.044504787772893906, 0.04394949972629547, 0.04373530298471451, 0.045798033475875854, 0.04392116144299507, 0.04295067861676216, 0.04336777329444885, 0.043398428708314896, 0.04247044026851654, 0.04311935231089592, 0.04263809323310852, 0.044295139610767365, 0.04145675152540207, 0.04135750234127045, 0.041512928903102875, 0.04232184216380119, 0.04114009067416191, 0.04185160994529724, 0.04160904884338379, 0.042306188493967056, 0.04219888895750046, 0.0408889465034008, 0.04142080992460251, 0.0423339307308197, 0.040743954479694366, 0.040178023278713226, 0.04099701717495918, 0.04238584265112877, 0.04077018424868584, 0.0413268581032753, 0.041181378066539764], [0.0592619925737381, 0.05449449643492699, 0.050241656601428986, 0.04807509481906891, 0.04758607596158981, 0.046418704092502594, 0.04533645883202553, 0.045337870717048645, 0.0463867224752903, 0.049408845603466034, 0.04369419813156128, 0.0433017835021019, 0.04429372027516365, 0.04290575534105301, 0.04330718517303467, 0.04279950261116028, 0.04267169162631035, 0.04246306046843529, 0.04225125163793564, 0.04189832881093025, 0.041548892855644226, 0.04411674290895462, 0.042524609714746475, 0.04163935035467148, 0.042164597660303116, 0.04155193641781807], [0.06057213991880417, 0.052804380655288696, 0.05029316991567612, 0.04766833409667015, 0.047947946935892105, 0.04580909013748169, 0.0463397316634655, 0.045403141528367996, 0.04462571069598198, 0.04521391540765762, 0.04356275126338005, 0.0436134934425354, 0.045293018221855164, 0.0442674346268177, 0.04433261603116989, 0.04461017623543739], [0.06128138676285744, 0.053548965603113174, 0.05206340551376343, 0.04816804453730583, 0.0502670481801033, 0.047234293073415756, 0.048370011150836945, 0.04621461406350136, 0.044402554631233215, 0.045161470770835876, 0.04418041929602623, 0.044561777263879776, 0.04344409704208374, 0.04346725717186928, 0.043347619473934174, 0.04403490945696831, 0.04243479296565056, 0.0427110381424427, 0.042557112872600555, 0.04221532493829727, 0.04315866529941559, 0.04176277294754982, 0.041689544916152954, 0.04174724221229553, 0.04162207245826721, 0.04168178141117096, 0.04151124507188797, 0.04159810394048691, 0.04126807302236557, 0.0411483496427536, 0.041064515709877014, 0.04103362187743187, 0.04112096503376961, 0.04142550379037857, 0.04181542992591858, 0.04059644043445587, 0.04197249189019203, 0.0410507507622242, 0.040798086673021317, 0.04241730645298958, 0.04126043990254402], [0.06231766194105148, 0.05340181291103363, 0.05094654858112335, 0.04898202791810036, 0.04765930771827698, 0.04637113958597183, 0.04546137526631355, 0.04496145620942116, 0.044924546033144, 0.0447232611477375, 0.04394269362092018, 0.04321794584393501, 0.043284498155117035, 0.04441148042678833, 0.04387761279940605, 0.04282551631331444, 0.04355712607502937, 0.0417570061981678, 0.04360835626721382, 0.042761124670505524, 0.042677100747823715, 0.04260179027915001, 0.04121316224336624, 0.04165777191519737, 0.04181293770670891, 0.04205774888396263, 0.041122425347566605, 0.040971748530864716, 0.042366765439510345, 0.04194389656186104, 0.04153919965028763, 0.04167018085718155, 0.04108242318034172]]\n",
            "[0.9150992035865784, 0.9126323461532593, 0.9159214496612549, 0.9135574102401733, 0.9095487594604492, 0.9163326025009155, 0.9167437553405762]\n",
            "[0.8545814156532288, 0.8513115048408508, 0.8603305816650391, 0.8515989780426025, 0.8462091088294983, 0.8542220592498779, 0.8539705276489258]\n",
            "[0.9880056381225586, 0.9881846904754639, 0.988264262676239, 0.9879459738731384, 0.9873597025871277, 0.9882516860961914, 0.9880810379981995]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oxLodYf6z6O0",
        "outputId": "4e2d1d98-3634-48b0-df17-49362b646211"
      },
      "source": [
        "print(\"Mean SAKHALIN test TOP-1 accuracy {}, std. dev {}\".format(np.mean(sakh_top_1), np.std(sakh_top_1)))\n",
        "print(\"Mean DAGESTAN test TOP-1 accuracy {}, std. dev {}\".format(np.mean(dag_top_1), np.std(dag_top_1)))\n",
        "print(\"Mean CALIFIRNIA test TOP-1 accuracy {}, std. dev {}\".format(np.mean(cal_top_1), np.std(cal_top_1)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mean SAKHALIN test TOP-1 accuracy 0.9142622181347438, std. dev 0.0023684848004979775\n",
            "Mean DAGESTAN test TOP-1 accuracy 0.8531748822757176, std. dev 0.003953111801177873\n",
            "Mean CALIFIRNIA test TOP-1 accuracy 0.9880132845469883, std. dev 0.00028922418772792563\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XOEVaNXH0D2T"
      },
      "source": [
        "### Plot convergence curves for latest iteration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aAI1ETjO5g0l",
        "outputId": "61d74bb9-609e-4699-c531-5add0ddf3769"
      },
      "source": [
        "# list all data in history\n",
        "print(history.history.keys())\n",
        "# summarize history for accuracy\n",
        "plt.plot(history.history['sparse_categorical_accuracy'])\n",
        "plt.plot(history.history['val_sparse_categorical_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper left')\n",
        "plt.show()\n",
        "# summarize history for loss\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dict_keys(['loss', 'sparse_categorical_accuracy', 'val_loss', 'val_sparse_categorical_accuracy'])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5ycdbn//9e1PduT3U3vpJAIIbRIJ4goSEcPFoKiP4VjOQcP6lc4dn761XN+Hg+2Y0dBQOCgFCUgBENRaiAVAkkgCdnNppBktve5fn987k0mS7KZSXa2ZN7Px2Mee8/d5pohzDWfcl+3uTsiIiLJyhroAEREZGhR4hARkZQocYiISEqUOEREJCVKHCIikhIlDhERSYkSh0gvzOx3ZvbtJPfdYGbvTndMIgNNiUNERFKixCGSAcwsZ6BjkMOHEocMeVEX0ZfMbIWZNZnZb8xslJk9ZGYNZrbIzIYn7H+Rmb1sZjEze9zMZiVsO9bMXoqOuwso6PFaF5jZsujYp81sTpIxnm9mS82s3sw2mdk3e2w/LTpfLNp+VbR+mJn9l5ltNLM6M/t7tG6+mVXv43N4d7T8TTO7x8xuM7N64Cozm2dmz0SvUWtmPzGzvITj32Fmj5rZTjPbamb/bmajzazZzCoS9jvOzLabWW4y710OP0occrh4P3AOMAO4EHgI+HegivDv/F8BzGwG8Afg89G2hcCfzSwv+hK9D/g9MAL43+i8RMceC9wMXANUAL8AHjCz/CTiawI+CpQD5wOfNrNLovNOiuL9cRTTXGBZdNz3geOBU6KY/g8QT/IzuRi4J3rN24Eu4N+ASuBk4GzgM1EMJcAi4GFgLDANeMzdtwCPA5cnnPdK4E5370gyDjnMKHHI4eLH7r7V3WuAp4Dn3H2pu7cC9wLHRvt9EHjQ3R+Nvvi+DwwjfDGfBOQCN7l7h7vfA7yQ8BpXA79w9+fcvcvdbwHaouN65e6Pu/tKd4+7+wpC8joz2vwRYJG7/yF63R3uvszMsoBPANe6e030mk+7e1uSn8kz7n5f9Jot7v6iuz/r7p3uvoGQ+LpjuADY4u7/5e6t7t7g7s9F224BFgCYWTbwYUJylQylxCGHi60Jyy37eF4cLY8FNnZvcPc4sAkYF22r8b0rf25MWJ4EfCHq6omZWQyYEB3XKzN7p5ktjrp46oB/JvzyJzrH6/s4rJLQVbavbcnY1COGGWb2FzPbEnVf/d8kYgC4H5htZlMIrbo6d3/+IGOSw4ASh2SazYQEAICZGeFLswaoBcZF67pNTFjeBHzH3csTHoXu/ockXvcO4AFggruXAT8Hul9nE3DEPo55C2jdz7YmoDDhfWQTurkS9Sx9/TPgVWC6u5cSuvISY5i6r8CjVtvdhFbHlai1kfGUOCTT3A2cb2ZnR4O7XyB0Nz0NPAN0Av9qZrlmdhkwL+HYXwH/HLUezMyKokHvkiRetwTY6e6tZjaP0D3V7Xbg3WZ2uZnlmFmFmc2NWkM3Az8ws7Fmlm1mJ0djKmuAguj1c4GvAgcaaykB6oFGMzsS+HTCtr8AY8zs82aWb2YlZvbOhO23AlcBF6HEkfGUOCSjuPtrhF/OPyb8or8QuNDd2929HbiM8AW5kzAe8qeEY5cAnwJ+AuwC1kX7JuMzwI1m1gB8nZDAus/7JvA+QhLbSRgYPyba/EVgJWGsZSfwH0CWu9dF5/w1obXUBOw1y2ofvkhIWA2EJHhXQgwNhG6oC4EtwFrgrITt/yAMyr/k7ondd5KBTDdyEpFkmNnfgDvc/dcDHYsMLCUOETkgMzsReJQwRtMw0PHIwFJXlYj0ysxuIVzj8XklDQG1OEREJEVqcYiISEoyovBZZWWlT548eaDDEBEZUl588cW33L3n9UGZkTgmT57MkiVLBjoMEZEhxcz2OfVaXVUiIpISJQ4REUmJEoeIiKQkI8Y49qWjo4Pq6mpaW1sHOpS0KigoYPz48eTm6p47ItI3MjZxVFdXU1JSwuTJk9m7GOrhw93ZsWMH1dXVTJkyZaDDEZHDRMZ2VbW2tlJRUXHYJg0AM6OiouKwb1WJSP/K2MQBHNZJo1smvEcR6V8Z21UlInLYaW+Guk0Q2wSxjWH5tH+DgrI+fRkljgESi8W44447+MxnPpPSce973/u44447KC8vT1NkIjKgOtuhswU626CjBTpbw6Ojde/1jdug7k2IvRklijeh+a29z5WVA0dfrsRxuIjFYvzP//zP2xJHZ2cnOTn7/8+ycOHCdIcmMvjEu6C+BlpiUDYeCkcc3Hm6OmDnenhrTThfVjZk5UJ2HmTnRo+8aF20XDXz4F+v266N8PK9sGsDtDdFj4Y9y22N0N4YluMdyZ83Ox/KJ0DZBDjy/LBcPik8L58AJWPCe+xjShwD5Prrr+f1119n7ty55ObmUlBQwPDhw3n11VdZs2YNl1xyCZs2baK1tZVrr72Wq6++GthTPqWxsZHzzjuP0047jaeffppx48Zx//33M2zYsAF+ZyKR5p2w7jHY9grkFYVfvfmlUFC652/3uvwSaNkVvmB3rQ/dLLs2Rn83QF01xDv3nLugDIZP3vejbAK0NcCOdSFBvLUG3lob/u7asPd5kmHZMPk0mHVheJSMTv79v3IfrLgb3nwmrCuqgrxiyCsmnltIR24p7fmjaC0tpMUKaPJ8mr2AzuwC4tl5xHMKiGflh7/ZBXh2HvHsYXhOPg3Zw9nhpTR1xGlq66SpvYvmzZ00ru+iub2TprYNNLW/zm+vOpEJIwp7jzVFGVFW/YQTTvCetapWr17NrFmzAPjWn1/mlc31ffqas8eW8o0L37Hf7Rs2bOCCCy5g1apVPP7445x//vmsWrVq97TZnTt3MmLECFpaWjjxxBN54oknqKio2CtxTJs2jSVLljB37lwuv/xyLrroIhYsWPC210p8ryJpE49D7VJYuwjWPQrVSwAPX7zelfr5Cith+KSQDMonheWC8pBEdm3Y84hthK72hAMtvG637DwYcQRUTofKGdHf6VA2ETwefuF3tUNXZ/gb7wgtk66O0DW08Wl45QHYsTace8I7YdaF+KwL2JE7hs2xFrbWt9HU1klLSxOVmxczZfODTNn1D7K9ky15k3im6GwW553JG50V7GrqoK6lg8a2FBNYL4blZlOUn01hXg5F+TkU5WVTmJ9DcX42Xz1/NmPLD+4HpZm96O4n9FyvFscgMW/evL2utfjRj37EvffeC8CmTZtYu3YtFRUVex0zZcoU5s6dC8Dxxx/Phg0b+i1e6QfxOLz0O3j8P6B8Ipz4SZh9MeQWHPw5O1pDF0xfdV8074TX/wZrH4V1i6I+doNxx8H862HaOTB2bviV31oPbfXQGktYrofWutBCGFYeJYjJ4f3mFycXQzwODZsTksnG0JqpiBJE+STIzol2dVo7u2jtiNPeGSfuTtwdB+JZTtygK+54thP3sLxr0lxqSq6ipfYVRlc/wswtjzN501ewR75CbXwyD3fN41WfyHuzlnB+9nOUWgtbvZxb/Fweyz2DTdnTKfQcisihqjiHGSNLKCvMZXhhHsMLcymL/g4vzKNsWC6lBeFi3e7Y4h6uyYr7nnXukJeTRWFeSBbZWf07e1KJA3ptGfSXoqKi3cuPP/44ixYt4plnnqGwsJD58+fv81qM/Pz83cvZ2dm0tLT0S6zSDzYvgwevg5oXYcJJ4Qv53qvh4evhuCvh+I/DiCQv6qyvhdcWwqsPwvonwSx8MQ+fDMOn7OniGTElfMkmfmG3N0H95j2Phu7l2jAwu/Xl8Kt92AiYdnZIFNPOhqLKvWPIyobiqvBIQTzubNzZzKu19aze0sCmnc10dMXpijudcSce/Q3Ps+mKT6UrPoW2zjitHV20dW6ktWM9bR1dtHZ20dF18D0sZjCq5HzGDn8/cwp3cnrXcxxd/wT/p+7uEGtuMW0zLqZlzuVUHnEmn8jJ4RMH/WqDmxLHACkpKaGhYd934ayrq2P48OEUFhby6quv8uyzz/ZzdHJQGreFPv11j4bujVFHwXEfhZnnhV/5yWitg799B174VeiquexXcPQ/hW3rn4AXfg1P/wT+8SOY9u7QCpl+zttbENvXwKt/CcmiJuqmHT4F5l0dfn3v2hAGiTc9H375JyqqComgcUuIp6eCcigdB6VjYOb7QrIYdxxkZdPa0cX2hja2vbWT7Q1tbG9oo6Gtk+L8HEoKcigtyKWkIJeSguj5sFyK83LIyjLqWjp4bUsDq2vreXVLPatrG3htSwMtHaGbK8tgTNkw8nOzyDYjO8vIyTays7LIyTKyzcjJyiI/xxhemEVBbjb5OVnk52ZTkJtFfk74W5CbTUFOFnk52WRnhWudsszIMsgyw6K/2VmGAcOL8hhXPoxRpQXk5SRe+nZh+FNXA9tfJWvSKQzLzYwxRiWOAVJRUcGpp57KUUcdxbBhwxg1atTubeeeey4///nPmTVrFjNnzuSkk04awEhlv7o6ofqF0EWz7lGoXR7WF1XBpFPDl/LdV0LRSJj7kZBEKo7Y97ncYeU98Nd/D62LEz8JZ30ldN90mzo/POo3w4u3wIu/gz98MPTVn3AVjJ8XYnn1wag/Hhh7LLzrq3DkBVB1ZPjZ3PN1W3bRueMNdlWvob52HZ1vvYG37KKh9GjqK6qozx1JXV4V9TlV1OdW0p5VgDs4Tqy2g+1r2tje8NTuJJEqMyjKy9mrz7+8MJcjR5fwoXkTmDW6lCPHlDB9ZAnD8vp+htAhKxsXHhlEg+MZ4LB7r+3N4ddww1Zo3Br6x9ubwnTGjuY9y+1NYd/2pjCd8pR/gQnzDu21m3eGL+Z1i+CNxeEXuWWH8057d3iMngNZWSGxrFsEL90Kax4OA8STT4fjPhZm53SPVWxfAwu/ELqRxh4HF/wgfOEfSFdHiOWFX8OGpwDwrBxax51My9TzaJ36HrpKwheaWfhl7e7U1rWyfnsTr7/VyPrtTbzxVhNv7mimvSu++9T5OVm7f3Gbhb8k/CLvXl82LJeq4nyqSvPD35J8RpaEv2G5gJKCkBQaWjupb+mgobWThtYO6lvDcn30vKokn1ljSpk1upRRpfmqejAI7G9wXIkjAwy599reDJueC/3nDbUhOTRsif5uhbZ9dJ90s+xoumPR3o9tq6FlJ0w5A874UvgCT+WLaevL8NwvwtTKzpYwP767T3/q/L1bBvtSXwvLbg9JJLYRhg2HOR+CnHx45qeQVwhnfwOOv2q/A9cdXXGqd7Ww/q1G3oi+8Ndvb2L9W00UNbzONNvMM/HZ1FO0z+N7ys02JlUUMbWyiClVRRxRWczUqiKmVBYxoihPX9yiWVUyiHW2h3749U/C+qeg+vk90ytzCqB4VJg7X3UkTD0LSkZB8eg9fwtK9ySL7Lx9J4T2JljyW3j6R3DLhWFK5RlfCi2E/X1BxrtCS+G5n4fYcgpgzgfhhI/DmLmpJZ7SMXDGF+G062hf9zidS35HwQu/ISveTs2kS3hp5nXs6iyj8cn1YU5+WxdNbZ00t3fR0NZJ9c5m3tzZTGd8zw+98sJcplYWceq0SqZWTWJ0aQHvNaJupMCjGUNEXUsAI0sKmFpVxLjyYeRkZ3S5OjlIanFkgEH3Xrs6w3jAhifDF/Kbz4YuJgzGzAmtgslnwPgTwi/zvvzl29EKS38P//hhqOMz5piQQGaeH7qXIFyItvQ2eP6XoYxD6XiY96kwRrGfK4hbO7rYWt/KlrpWttS3sq2+jbca29jR1M7OpnZ2JCw3t4fB3uHUU2ZNbPAxe50ry6AoP4fi/BwK87Ipzs9hbPmwqDVQzJTK0EoYXpTXd5+LyD6oxSGpcw9dRFtWhKt/LSv8qs/t7gIqDL/0cwv3dAl5PHQpNW6L/m7dMxbRva5hS+juAaiaBcdeCVNODwPKh1ra4UByC6Ik8DFYcRf8/Qdw14IQx0mfhtplsPzOkMgmnQrv+TbMPJ/GTnh9WyPrVlezcUcTW+pb2VLfxta6VrY2tBJrfnuZiLycLCqK8qgozmNEUT5Tq4oZUZTHiKI8KqN15YW5FOfnUJSXQ1F+NkX5OeTnZKmbSAY1JQ4J3MMUzdrlIVHULofaFdC07dDPXVAeupuKR8K440O309hjQ8uieOShn/9g5OSF6yGO+XCoIfTU9+HP/4pn57N9ykUsGfVPvNA6nnXPNPL6/U+wuW7PdTRZBpXF+YwuK2BiRSHzpoxgdFkBo0oLGF1awOiyfEaWFlCSn6MEIIclJY5M1RILA9Abnw4XmdWu2DPonJUTxhOmnxNmCI05Bka9Iwza7i7Q1hTNYEqcvdQYupWKR+9JFMUjwwDwAOuKOzsa29hS38rW+ja21rcmPCaxrf0/GGkrWdFUya5VpbCqk2G5m5g2sph3Tq1g2sji3Y+JIwrJ1diAZDAljgFysGXVAW666SauvvpqCgtTKFzWsCUkiTefgY3PwNZVgIcqoGPmwNHvDwli9BwYOXv/ZS3ykpuxM1DcnZpYC6trG6Krjet5tbaBjTub6YrvPZ6X2HIYN6KY0VPmc3pF0e4EMbZsGFn9XMpBZChQ4hgg+yurnoybbrqJBQsW9J44PB5aFW0NYSrof0UXEeYWwYQTYf4NMOmU0HWU17eVM/tDR1ecbQ1tbKlrYc3WxnC1cW0Dq7fU09C650KySRWFHDm6hHOPGs2YqDtpVGkBo8sKqCjK06wikYOgxDFAEsuqn3POOYwcOZK7776btrY2Lr30Ur71rW/R1NTE5ZdfTnV1NV1dXXzta19j69atbN68mbPOOovKykoWL1789pO3NYQKop2t4bqG7NwwyDvxlNC6SLb8xQBxd6p3tbB2WwNb69vYUtfKtobWvZbfamzf65ji/ByOHF3CxXPHMmtMKUeOLmXm6BKK8/VPXKSv6f8qgIeuhy0r+/aco4+G8763383f+973WLVqFcuWLeORRx7hnnvu4fnnn8fdueiii3jyySfZvn07Y8eO5cEHHwRCDauysjJ+8IMfsHjxYiorexSS62wLN6dprQvXMwyfEu5bEHsVTviXvn1/fWhnUzvLq2Ms3xQ9quvY2bQnMZhBRVE+o0pDt9IxE8rDctR6mDaymPHDh2kgWqSfKHEMAo888giPPPIIxx4bykw0Njaydu1aTj/9dL7whS/w5S9/mQsuuIDTTz993yeIx6EpmvZqFq5qLhq557qEQaS7kN2K6pAglm+K8ebOZiCEPmNkCe+eNZJjJpRz5OhSxpQVUFWSr8FokUFEiQN6bRn0B3fnhhtu4JprrnnbtpdeeomFCxfy1a9+lbPPPpuvf/3riQeG1kV9TbjSurtqac7AXxjW0t7Fum2NvLa1gTVbQ5XTNVsbqE2Y1jqufBjHTCjjindO5JgJ5Rw1rkxdSyJDgP4vHSCJZdXf+9738rWvfY0rrriC4uJiampqyM3NpbOzkxEjRrBgwQLKy8v59a9/vefYnW9RSSzctzinACqmhdtv9rN4PIxHvFJbxyu1Dby2pZ7XtoRZTN1FCfJyspg+spiTp1YwY3QJM0YVc9S4MkaWHMINiURkwChxDJDEsurnnXceH/nIRzj55JMBKC4u5rbbbmPdunV86UtfIisri9zcXH72059AS4yrF7yfc887l7Gjq1j814Xhpjn90L/f0t7Fa1vD/RJe2Vwf3TehYXc57CyDyZVFzB5byiXHjmPmqBJmjC5h0ohCzV4SOYyoVtVg19UZLsxriW6vSTzMlBo2PFyBncQMqYN9r+7Oy5vrWbiylkWrt7JuWyPdl0IU5+cwa0wJs8aUMntMKbPGlDJj1CC9X4KIHBTVqhpKOtvD2EVrLFyNDeFCvaIRYRwjryjUjUoDd2dFdR0LV9Xy0MotvLmzmews46SpIzjvqDHMGlPKO8aWahaTSAZT4hgM4l1R2Y6G0KroiAoAdpcULygLhQTT9EUdjzvLqmM8tLKWhSu3UBNrISfLOGVaJZ896wjOmT2aEarEKiKRjE4c7j4wv5o9HtV2aoC2qNYTDli4irtkbJQsDn3weH9dkfG4s3RTjAdX1PLQqlpq61rJzTZOm1bJ5989nXNmj6K8UMlCRN4uYxNHQUEBO3bsoKKion+SR1d7uM9DW2PofvLoNp25w8I9qvNLQhfUfu7+djDcnR07dlBQULD7+e5ksbKWzXWt5GVnccaMSr703pmcPWsUZcMG91XlIjLw0po4zOxc4IdANvBrd/9ej+2TgJuBKmAnsMDdq6Nt/wmcD2QBjwLXurub2ePAGCDqz+E97p5y7e/x48dTXV3N9u3bD+q9pSTeGe5FEe8Mg9k5BaFibHYBZDlQHz36XkFBAbushDsefGV3N1R3svjie2fy7tmjKC1QshCR5KUtcZhZNvBT4BygGnjBzB5w91cSdvs+cKu732Jm7wK+C1xpZqcApwJzov3+DpwJPB49v8Ld954mlaLc3FymTJlyKKdITksMbj431I762AMw7ri0v6S780ptPfcv28yDK9ZTE2shN9s4Y3oVX3jPDCULETkk6WxxzAPWufsbAGZ2J3AxkJg4ZgPXRcuLgfuiZQcKgDzAgFxgaxpjTY/ONrjzCtixDhbck/akURNr4f5lNdy3tIY1WxvJzTZOn17FdeeEZKFuKBHpC+lMHOOATQnPq4F39thnOXAZoTvrUqDEzCrc/RkzWwzUEhLHT9x9dcJxvzWzLuCPwLd9HyPAZnY1cDXAxIkT++gtpSAeh3uvgY1/h8t+BVPnp+Vl6lo6eGhlLfcureG59TsBOHHycL5z6VGcf/QYDXCLSJ8b6MHxLwI/MbOrgCeBGqDLzKYBs4Dx0X6Pmtnp7v4UoZuqxsxKCInjSuDWnid2918Cv4RwAWDa30lPj3w13JL03d+COZf36anbO+Msfm0b9y2t4bFXt9HeGWdqVRFfOGcGF88dx8SKoXd/DREZOtKZOGqACQnPx0frdnP3zYQWB2ZWDLzf3WNm9ingWXdvjLY9BJwMPOXuNdGxDWZ2B6FL7G2JY0A9/RN49qcw7xo49do+O217Z5y7l2zip4vXUVvXSmVxHle8cyKXHjuOo8eV6YI8EekX6UwcLwDTzWwKIWF8CPhI4g5mVgnsdPc4cANhhhXAm8CnzOy7hK6qM4GbzCwHKHf3t8wsF7gAWJTG95C6lffAI1+BWRfBud/tk4v2Orri/PHFan78t3XUxFo4bmI5377kKM6cUaUaUCLS79KWONy908w+B/yVMB33Znd/2cxuBJa4+wPAfOC7ZuaErqrPRoffA7wLWEkYKH/Y3f9sZkXAX6OkkU1IGr9K13tI2fqn4L5Pw8STw7jGIV6T0dkV596lNfzob2vZtLOFYyaU838vO5ozpleqdSEiAyZjixz2ua0vw83nhcKDn3gYCkcc9Km64s4Dy2v44aK1bNjRzFHjSrnunBmcNXOkEoaI9BsVOUynumq47QOhXMiCPx500nB3/rKilv9etIY3tjcxa0wpv7zyeM6ZPUoJQ0QGDSWOQ9XeFJJGeyN8/CEon3DgY/ahK+5884GX+f2zG5k5qoSfLziO98weTVaWEoaIDC5KHIdq1R9h+2r4yN0w+qiDOkVrRxfX3b2MhSu3cM2ZU/nye49UwhCRQUuJ41AtvQ0qZ8L09xzU4fWtHXzqliU8t34nXz1/Fp88fWofBygi0reUOA7F9jWw6Tk45/89qGm32+pb+dhvX2DdtgZ++KG5XDx3XBqCFBHpW0och2LZbeE2rnM+mPKhb2xv5KM3P8/OpnZ+87ETOWNGVRoCFBHpe0ocB6urE5bfCTPeCyWjUjp0+aYYH//dCxhw59UnMWd8eXpiFBFJAyWOg7VuETRuhWMXpHTYE2u28+nbXqSiOI9bP/FOplQWpSlAEZH0UOI4WEt/H+7cl8Kg+L1Lq/nS/65gxqgSfveJExlZcui3hhUR6W8qdHQwmt6CNQ+HsY3s5O5xccdzb/Jvdy3nxMkjuOuak5Q0RGTIUovjYKy4K9wGNsluqupdzdz4l5c5Y0YVv/ro8eTn9N19xUVE+ptaHKlyh5d+D+NOgJGzkjrk239ZjWF897KjlTREZMhT4kjV5pfCleLHXpHU7k+u2c7DL2/hc++axrjyYWkOTkQk/ZQ4UrX0dsgpgKPef8Bd2zvjfPPPLzO5opBPnj6lH4ITEUk/jXGkoqMl3Khp9sVQUHbA3X/z9/W8sb2J3378RHVRichhQy2OVKz+C7TVwdwDd1PV1rXw47+t5ZzZozhr5sh+CE5EpH8ocaRi6e+hfCJMPv2Au377wdV0xZ2vXzC7HwITEek/ShzJ2rUR1j8JcxdAVu8f2z/WvcWDK2r5zPxpTBhR2E8Bioj0DyWOZC3/Q/g798O97tbeGecbD7zMxBGFXHOmSqSLyOFHiSMZ8XiYTTX1zNBV1YvfPb2eddsa+caFsynI1YC4iBx+lDiSseEpqHsTjr2y19221rfyw0VrOfvIkZw9K7WKuSIiQ4USRzKW3ham3x55fq+7fefB1XTEnW9c+I5+CkxEpP8pcRxISwxWPwBHfQBy93/l9zOv7+CB5Zv55zOPYGKFBsRF5PClxHEgq/4Ina29FjTs6IrzjQdWMX74MD4z/4h+DE5EpP8pcRzIstth5Dtg7LH73eWWpzewZmsjX79AA+IicvhT4ujN1leg5sVQ0NBsn7vEmtu5adFa5s+s4pzZGhAXkcOfEkdvlt0OWTnhhk37sWZrI41tnVx1ymRsP8lFRORwosTRm652eMelUFS5311ize0AVBbn91dUIiIDStVxe/O+/y/cuKkXsZYOAMqGJXcLWRGRoU4tjgM5QPdTXXNIHOWFShwikhmUOA5RrKWd7CyjOF+NNxHJDEochyjW3EH5sFwNjItIxlDiOESxlg7K1E0lIhlEieMQ1UUtDhGRTKHEcYhiLe2UF+YNdBgiIv1GieMQxdTiEJEMo8RxiOqaNcYhIplFieMQdHTFaWjrpHyYuqpEJHOkNXGY2blm9pqZrTOz6/exfZKZPWZmK8zscTMbn7DtP83sZTNbbWY/smi+q5kdb2Yro3PuXj8Q6lt08Z+IZJ60JQ4zywZ+CpwHzAY+bGaze+z2feBWd58D3Ah8Nzr2FOBUYA5wFGxRiPAAABJpSURBVHAicGZ0zM+ATwHTo8e56XoPBxJT4hCRDJTOFsc8YJ27v+Hu7cCdwMU99pkN/C1aXpyw3YECIA/IB3KBrWY2Bih192fd3YFbgUvS+B56FWtWnSoRyTzpTBzjgE0Jz6ujdYmWA5dFy5cCJWZW4e7PEBJJbfT4q7uvjo6vPsA5ATCzq81siZkt2b59+yG/mX3proyr6bgikkmSShxm9iczO9/M+jrRfBE408yWErqiaoAuM5sGzALGExLDu8zs9FRO7O6/dPcT3P2EqqqqPg476G5xaDquiGSSZBPB/wAfAdaa2ffMbGYSx9QAExKej4/W7ebum939Mnc/FvhKtC5GaH086+6N7t4IPAScHB0/vrdz9ieNcYhIJkoqcbj7Ine/AjgO2AAsMrOnzezjZra/b80XgOlmNsXM8oAPAQ8k7mBmlQmtmBuAm6PlNwktkZzo/GcCq929Fqg3s5Oi2VQfBe5P+t32sbrmdsygtECJQ0QyR9JdT2ZWAVwFfBJYCvyQkEge3df+7t4JfA74K7AauNvdXzazG83somi3+cBrZrYGGAV8J1p/D/A6sJIwDrLc3f8cbfsM8GtgXbTPQ8m+h74Wa+mgbFguWVmqjCsimSOpm0iY2b3ATOD3wIXRL3+Au8xsyf6Oc/eFwMIe676esHwPIUn0PK4LuGY/51xCmKI74FRuREQyUbJ3H/qRuy/e1wZ3P6EP4xlSQkl1zagSkcySbFfVbDMr735iZsPN7DNpimnIqGtuV4tDRDJOsonjU9FsJwDcfRfh6u2MFmvp0IwqEck4ySaO7MSaUFE5kYzvo9EYh4hkomTHOB4mDIT/Inp+TbQuY3XFnfpWjXGISOZJNnF8mZAsPh09f5QwJTZjNbR24K6rxkUk8ySVONw9TqhK+7P0hjN07C43ojEOEckwyV7HMZ1Q8nw2oWotAO4+NU1xDXoqNyIimSrZwfHfElobncBZhHLmt6UrqKGguzJume7+JyIZJtnEMczdHwPM3Te6+zeB89MX1uBXpxaHiGSoZAfH26JihGvN7HOEirTF6Qtr8FNJdRHJVMm2OK4FCoF/BY4HFgAfS1dQQ4Hu/icimeqALY7oYr8PuvsXgUbg42mPagiItbRTkp9DTnY6b6IoIjL4HPBbL6pUe1o/xDKk1DV3UKbxDRHJQMmOcSw1sweA/wWaule6+5/SEtUQoDpVIpKpkk0cBcAO4F0J6xzI3MTR3E65puKKSAZK9spxjWv0EGvpYEz5sIEOQ0Sk3yV75fhvCS2Mvbj7J/o8oiGiTpVxRSRDJdtV9ZeE5QLgUmBz34czNLi7xjhEJGMl21X1x8TnZvYH4O9piWgIaGjrpCvuGuMQkYx0sBchTAdG9mUgQ0mdKuOKSAZLdoyjgb3HOLYQ7tGRkfaUVFeLQ0QyT7JdVSXpDmQoibWEyrhqcYhIJkqqq8rMLjWzsoTn5WZ2SfrCGtxU4FBEMlmyYxzfcPe67ifuHgO+kZ6QBr/umzip5IiIZKJkE8e+9kt2Ku9hp273TZyUOEQk8ySbOJaY2Q/M7Ijo8QPgxXQGNpjFmjsozMsmPyd7oEMREel3ySaOfwHagbuAO4FW4LPpCmqwi7XoqnERyVzJzqpqAq5PcyxDRqy5gzJNxRWRDJXsrKpHzaw84flwM/tr+sIa3Opa2tXiEJGMlWxXVWU0kwoAd99FBl85HmtWnSoRyVzJJo64mU3sfmJmk9lHtdxMoQKHIpLJkp1S+xXg72b2BGDA6cDVaYtqEHP3cNtYFTgUkQyV7OD4w2Z2AiFZLAXuA1rSGdhg1dLRRXtXXC0OEclYyRY5/CRwLTAeWAacBDzD3reSzQgqNyIimS7ZMY5rgROBje5+FnAsEOv9kMNTTCXVRSTDJZs4Wt29FcDM8t39VWBm+sIavLor42qMQ0QyVbKJozq6juM+4FEzux/YeKCDzOxcM3vNzNaZ2dsuIDSzSWb2mJmtMLPHzWx8tP4sM1uW8GjtrsZrZr8zs/UJ2+Ym/3YPnW7iJCKZLtnB8UujxW+a2WKgDHi4t2PMLBv4KXAOUA28YGYPuPsrCbt9H7jV3W8xs3cB3wWudPfFwNzoPCOAdcAjCcd9yd3vSSb2vtZdGVeJQ0QyVcoVbt39iSR3nQesc/c3AMzsTuBiIDFxzAaui5YXE1o0PX0AeMjdm1ONNR32DI6rq0pEMtPB3nM8GeOATQnPq6N1iZYDl0XLlwIlZlbRY58PAX/ose47UffWf5tZ/r5e3MyuNrMlZrZk+/btB/cO9iHW0k5eThYFuen86EREBq+B/vb7InCmmS0FzgRqgK7ujWY2BjgaSKyLdQNwJGGW1wj2c+9zd/+lu5/g7idUVVX1WcB1zaEyrpn12TlFRIaSdN6MqQaYkPB8fLRuN3ffTNTiMLNi4P2JNbGAy4F73b0j4ZjaaLHNzH5LSD79RnWqRCTTpbPF8QIw3cymmFkeocvpgcQdzKzSzLpjuAG4ucc5PkyPbqqoFYKFn/yXAKvSEPt+7Wpu1/iGiGS0tCUOd+8EPkfoZloN3O3uL5vZjWZ2UbTbfOA1M1sDjAK+0318VEhxAtBzMP52M1sJrAQqgW+n6z3sS50KHIpIhkvrfcPdfSGwsMe6rycs3wPsc1qtu2/g7YPpuPuAljmJNXcwZ7wSh4hkroEeHB9yYi3tlOvufyKSwZQ4UtDa0UVrR5wyFTgUkQymxJGCOl01LiKixJEKXTUuIqLEkZJYc6iMqxaHiGQyJY4UdBc41BiHiGQyJY4UqKS6iIgSR0q6b+Kk6bgiksmUOFIQa+4gJ8soysse6FBERAaMEkcKYlG5EVXGFZFMpsSRgrrmDg2Mi0jGU+JIgcqNiIgocaQkFt3ESUQkkylxpCDW3EGZpuKKSIZT4khBXUuHyo2ISMZT4khSR1ecxrZOXfwnIhlPiSNJqowrIhIocSSpuzKupuOKSKZT4khSncqNiIgAShxJ23MvDrU4RCSzKXEkKabKuCIigBJH0nY1q6tKRASUOJJW19JBlkFJfs5AhyIiMqCUOJIUiwocZmWpMq6IZDYljiSFkurqphIRUeJIUqy5XddwiIigxJG0uugmTiIimU6JI0kqqS4iEihxJCnWrJs4iYiAEkdSuuJOfWunxjhERFDiSEq9KuOKiOymxJGEmBKHiMhuShxJiHWXG9Hd/0RElDiS0d3i0P3GRUSUOJJSp5LqIiK7KXEkIabKuCIiuylxJKG7q6q0QJVxRUSUOJIQa+6gpCCHnGx9XCIiaf0mNLNzzew1M1tnZtfvY/skM3vMzFaY2eNmNj5af5aZLUt4tJrZJdG2KWb2XHTOu8ws7f1HqlMlIrJH2hKHmWUDPwXOA2YDHzaz2T12+z5wq7vPAW4Evgvg7ovdfa67zwXeBTQDj0TH/Afw3+4+DdgF/D/peg/dYs3tmoorIhJJZ4tjHrDO3d9w93bgTuDiHvvMBv4WLS/ex3aADwAPuXuzmRkhkdwTbbsFuKTPI+8hphaHiMhu6Uwc44BNCc+ro3WJlgOXRcuXAiVmVtFjnw8Bf4iWK4CYu3f2ck4AzOxqM1tiZku2b99+kG8hqIvu/iciIgM/OP5F4EwzWwqcCdQAXd0bzWwMcDTw11RP7O6/dPcT3P2EqqqqQwpSLQ4RkT3SOb+0BpiQ8Hx8tG43d99M1OIws2Lg/e4eS9jlcuBed++Inu8Ays0sJ2p1vO2cfS0ed41xiIgkSGeL4wVgejQLKo/Q5fRA4g5mVmlm3THcANzc4xwfZk83Fe7uhLGQD0SrPgbcn4bYd2ts7yTuKnAoItItbYkjahF8jtDNtBq4291fNrMbzeyiaLf5wGtmtgYYBXyn+3gzm0xosTzR49RfBq4zs3WEMY/fpOs9AMSauivjqsUhIgLp7arC3RcCC3us+3rC8j3smSHV89gN7GPg293fIMzY6hexlu7KuGpxiIjAwA+OD3qxZt2LQ0QkkRLHAegmTiIie1PiOIC6qDJumWZViYgAShwH1N1VpQsARUQCJY4DiLV0UJSXTV6OPioREVDiOKBYc4em4oqIJFDiOIC6lnZ1U4mIJFDiOIDQ4lDiEBHppsRxACpwKCKyNyWOA4g1d2gqrohIAiWOXrg7dS3tanGIiCRQ4uhFc3sXHV2uOlUiIgmUOHqhciMiIm+nxNGLmMqNiIi8jRJHL+pUGVdE5G2UOHqhrioRkbdT4ujF7ntxqKtKRGQ3JY5e7L77n1ocIiK7KXH0oq65g/ycLApyswc6FBGRQUOJoxeqUyUi8nZKHL2ItbRrfENEpIecgQ5gMJszvpypVcUDHYaIyKCixNGLz541baBDEBEZdNRVJSIiKVHiEBGRlChxiIhISpQ4REQkJUocIiKSEiUOERFJiRKHiIikRIlDRERSYu4+0DGknZltBzYe5OGVwFt9GE5/G8rxD+XYYWjHP5Rjh6Ed/2CKfZK7V/VcmRGJ41CY2RJ3P2Gg4zhYQzn+oRw7DO34h3LsMLTjHwqxq6tKRERSosQhIiIpUeI4sF8OdACHaCjHP5Rjh6Ed/1COHYZ2/IM+do1xiIhIStTiEBGRlChxiIhISpQ4emFm55rZa2a2zsyuH+h4UmFmG8xspZktM7MlAx3PgZjZzWa2zcxWJawbYWaPmtna6O/wgYxxf/YT+zfNrCb6/JeZ2fsGMsbemNkEM1tsZq+Y2ctmdm20ftB//r3EPiQ+fzMrMLPnzWx5FP+3ovVTzOy56LvnLjMbVPew1hjHfphZNrAGOAeoBl4APuzurwxoYEkysw3ACe4+WC4k6pWZnQE0Are6+1HRuv8Edrr796LEPdzdvzyQce7LfmL/JtDo7t8fyNiSYWZjgDHu/pKZlQAvApcAVzHIP/9eYr+cIfD5m5kBRe7eaGa5wN+Ba4HrgD+5+51m9nNgubv/bCBjTaQWx/7NA9a5+xvu3g7cCVw8wDEdttz9SWBnj9UXA7dEy7cQvhAGnf3EPmS4e627vxQtNwCrgXEMgc+/l9iHBA8ao6e50cOBdwH3ROsH3WevxLF/44BNCc+rGUL/IAn/+B4xsxfN7OqBDuYgjXL32mh5CzBqIIM5CJ8zsxVRV9ag6+bZFzObDBwLPMcQ+/x7xA5D5PM3s2wzWwZsAx4FXgdi7t4Z7TLovnuUOA5fp7n7ccB5wGej7pQhy0Of6lDqV/0ZcAQwF6gF/mtgwzkwMysG/gh83t3rE7cN9s9/H7EPmc/f3bvcfS4wntDTceQAh3RAShz7VwNMSHg+Plo3JLh7TfR3G3Av4R/kULM16sPu7sveNsDxJM3dt0ZfCHHgVwzyzz/qX/8jcLu7/ylaPSQ+/33FPtQ+fwB3jwGLgZOBcjPLiTYNuu8eJY79ewGYHs1uyAM+BDwwwDElxcyKooFCzKwIeA+wqvejBqUHgI9Fyx8D7h/AWFLS/YUbuZRB/PlHA7S/AVa7+w8SNg36z39/sQ+Vz9/MqsysPFoeRpiMs5qQQD4Q7TboPnvNqupFNIXvJiAbuNndvzPAISXFzKYSWhkAOcAdgz12M/sDMJ9QUnor8A3gPuBuYCKhLP7l7j7oBqH3E/t8QjeJAxuAaxLGCwYVMzsNeApYCcSj1f9OGCsY1J9/L7F/mCHw+ZvZHMLgdzbhh/zd7n5j9P/wncAIYCmwwN3bBi7SvSlxiIhIStRVJSIiKVHiEBGRlChxiIhISpQ4REQkJUocIiKSEiUOkUHOzOab2V8GOg6RbkocIiKSEiUOkT5iZguieyssM7NfRMXrGs3sv6N7LTxmZlXRvnPN7NmoCN+93UX4zGyamS2K7s/wkpkdEZ2+2MzuMbNXzez26IppkQGhxCHSB8xsFvBB4NSoYF0XcAVQBCxx93cATxCuKge4Ffiyu88hXPXcvf524KfufgxwCqFAH4Sqr58HZgNTgVPT/qZE9iPnwLuISBLOBo4HXogaA8MIRQHjwF3RPrcBfzKzMqDc3Z+I1t8C/G9UX2ycu98L4O6tANH5nnf36uj5MmAy4aY/Iv1OiUOkbxhwi7vfsNdKs6/12O9ga/wk1inqQv/vygBSV5VI33gM+ICZjYTd9+ueRPh/rLvK6UeAv7t7HbDLzE6P1l8JPBHdwa7azC6JzpFvZoX9+i5EkqBfLSJ9wN1fMbOvEu66mAV0AJ8FmoB50bZthHEQCKWyfx4lhjeAj0frrwR+YWY3Ruf4p358GyJJUXVckTQys0Z3Lx7oOET6krqqREQkJWpxiIhIStTiEBGRlChxiIhISpQ4REQkJUocIiKSEiUOERFJyf8PH1sgV2DYU/gAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhcZZn///fdS3V3dTrdnU5n66xAgIQEAsQAoohGNKCACLIJ4zaCC5fOb/w6gOMG8x3Fcf26jeCAg4wKDAhGDQIKqMiWEJaskBBD0klIOr2l9/X+/fGcTipNdehOurqqU5/XdfV1Tp1zqurugtSnz/Oc8zzm7oiIiPSXk+4CREQkMykgREQkKQWEiIgkpYAQEZGkFBAiIpKUAkJERJJSQIgMAzP7bzP7v4M8drOZvfNQX0ck1RQQIiKSlAJCRESSUkBI1oiadj5vZi+aWYuZ3WpmE83sATNrMrM/mll5wvHnmdkaM2sws8fMbE7CvhPNbGX0vLuAwn7v9V4zez567hNmdvxB1vxxM9toZnVmttTMpkTbzcy+a2a7zGyPma0ys3nRvnPMbG1U2zYz+z8H9YFJ1lNASLa5EDgLOBo4F3gA+AJQSfj38BkAMzsa+BXwT9G+ZcBvzSxmZjHgfuAOYBzwv9HrEj33ROA24GqgArgZWGpmBUMp1MzeAXwduBiYDLwK3BntfhdwRvR7lEbH1Eb7bgWudvcSYB7wyFDeV6SPAkKyzQ/cfae7bwP+Cjzt7s+5eztwH3BidNwlwO/d/WF37wK+BRQBbwZOBfKB77l7l7vfAyxPeI+rgJvd/Wl373H324GO6HlD8UHgNndf6e4dwPXAaWY2E+gCSoBjAXP3de6+I3peFzDXzMa6e727rxzi+4oACgjJPjsT1tuSPB4TrU8h/MUOgLv3AluBqmjfNt9/pMtXE9ZnAJ+LmpcazKwBmBY9byj619BMOEuocvdHgB8CPwJ2mdktZjY2OvRC4BzgVTP7s5mdNsT3FQEUECID2U74ogdCmz/hS34bsAOoirb1mZ6wvhX4d3cvS/iJu/uvDrGGYkKT1TYAd/++u58MzCU0NX0+2r7c3c8HJhCawu4e4vuKAAoIkYHcDbzHzBabWT7wOUIz0RPAk0A38Bkzyzez9wOLEp77U+ATZnZK1JlcbGbvMbOSIdbwK+AjZrYg6r/4GqFJbLOZvSl6/XygBWgHeqM+kg+aWWnUNLYH6D2Ez0GymAJCJAl3fwm4AvgBsJvQoX2uu3e6eyfwfuDDQB2hv+LXCc9dAXyc0ARUD2yMjh1qDX8EvgTcSzhrORK4NNo9lhBE9YRmqFrgm9G+K4HNZrYH+AShL0NkyEwTBomISDI6gxARkaQUECIikpQCQkREklJAiIhIUnnpLmC4jB8/3mfOnJnuMkRERpVnn312t7tXJtt32ATEzJkzWbFiRbrLEBEZVczs1YH2qYlJRESSSmlAmNkSM3spGq74uiT7z4iGTO42s4v67ftDNI7N71JZo4iIJJeygDCzXMJAYmcTxoq5zMzm9jtsC+EO018meYlvEu4IFRGRNEhlH8QiYKO7bwIwszuB84G1fQe4++Zo3+vGinH3P5nZmYdSQFdXF9XV1bS3tx/Ky4wKhYWFTJ06lfz8/HSXIiKHiVQGRBVhVMs+1cApw/kGZnYVYex9pk+f/rr91dXVlJSUMHPmTPYfePPw4u7U1tZSXV3NrFmz0l2OiBwmRnUntbvf4u4L3X1hZeXrr9Jqb2+noqLisA4HADOjoqIiK86URGTkpDIgthHGz+8zNdo2og73cOiTLb+niIycVAbEcmC2mc2K5vC9FFiawvc7KD29vezc005rZ3e6SxERySgpCwh37wauAR4E1gF3u/saM7vRzM4DiCY9qQY+ANxsZmv6nm9mfyVMBr/YzKrN7N2pqRN27mmnpaMnFS9PQ0MDP/7xj4f8vHPOOYeGhoYUVCQiMjgpvZPa3ZcBy/pt+3LC+nJC01Oy5741lbX1yc0xDOjpTc28GH0B8alPfWq/7d3d3eTlDfzxL1u2bMB9IiIj4bAZauNgmRm5OTn09KZmVsbrrruOV155hQULFpCfn09hYSHl5eWsX7+el19+mfe9731s3bqV9vZ2PvvZz3LVVVcB+4YOaW5u5uyzz+Ytb3kLTzzxBFVVVfzmN7+hqKgoJfWKiPTJmoC44bdrWLt9T9J9bZ095ORAQV7ukF5z7pSxfOXc4w54zE033cTq1at5/vnneeyxx3jPe97D6tWr916OettttzFu3Dja2tp405vexIUXXkhFRcV+r7FhwwZ+9atf8dOf/pSLL76Ye++9lyuuuGJItYqIDFXWBMSBmIW+iJGwaNGi/e5V+P73v899990HwNatW9mwYcPrAmLWrFksWLAAgJNPPpnNmzePTLEiktWyJiAO9Jf+5t0tdPb0cvTEkpTXUVxcvHf9scce449//CNPPvkk8XicM888M+m9DAUFBXvXc3NzaWtrS3mdIiKj+ka54ZKbYynrpC4pKaGpqSnpvsbGRsrLy4nH46xfv56nnnoqJTWIiByMrDmDOJC8XKO713H3Yb/hrKKigtNPP5158+ZRVFTExIkT9+5bsmQJP/nJT5gzZw7HHHMMp5566rC+t4jIoTAfqcb3FFu4cKH3nzBo3bp1zJkz5w2fu6upndca2zluSim5OaP3juTB/r4iIn3M7Fl3X5hsn5qYgLwoFFLVzCQiMhopIIDcnPAxpOpeCBGR0UgBwb4ziG6dQYiI7KWAgL39DmpiEhHZRwHBvoDQGYSIyD4KCNRJLSKSjAKCvgH7UnOz3MEO9w3wve99j9bW1mGuSERkcBQQkdwco7tHASEi0kd3UkfycnLoTsFlronDfZ911llMmDCBu+++m46ODi644AJuuOEGWlpauPjii6murqanp4cvfelL7Ny5k+3bt/P2t7+d8ePH8+ijjw57bSIiB5I9AfHAdfDaqgF3V3X14DjkD+EjmTQfzr7pgIckDvf90EMPcc899/DMM8/g7px33nn85S9/oaamhilTpvD73/8eCGM0lZaW8p3vfIdHH32U8ePHD74mEZFhoiamiBmQ4j7qhx56iIceeogTTzyRk046ifXr17Nhwwbmz5/Pww8/zLXXXstf//pXSktLU1uIiMggZM8ZxBv8pV/X0EZdSyfzqlL35ezuXH/99Vx99dWv27dy5UqWLVvGF7/4RRYvXsyXv/zlJK8gIjJydAYRyc0xet3pHebBCxOH+373u9/NbbfdRnNzMwDbtm1j165dbN++nXg8zhVXXMHnP/95Vq5c+brnioiMtJQGhJktMbOXzGyjmV2XZP8ZZrbSzLrN7KJ++z5kZhuinw+lsk5I3b0QicN9P/zww1x++eWcdtppzJ8/n4suuoimpiZWrVrFokWLWLBgATfccANf/OIXAbjqqqtYsmQJb3/724e1JhGRwUjZcN9mlgu8DJwFVAPLgcvcfW3CMTOBscD/AZa6+z3R9nHACmAhoWfgWeBkd68f6P0OZbhvgIbWTrbUtTJ7YglF+UObmzpTaLhvERmqdA33vQjY6O6b3L0TuBM4P/EAd9/s7i8C/a8vfTfwsLvXRaHwMLAkhbXuO4NIwb0QIiKjUSoDogrYmvC4Oto2bM81s6vMbIWZraipqTnoQkFDfouI9DeqO6nd/RZ3X+juCysrKwc6ZlCvNdqH/D5cZgYUkcyRyoDYBkxLeDw12pbq5+5VWFhIbW3toL48R/OQ3+5ObW0thYWF6S5FRA4jqbwPYjkw28xmEb7cLwUuH+RzHwS+Zmbl0eN3AdcPtYCpU6dSXV3NYJufahraaC3Io7Yof6hvlXaFhYVMnTo13WWIyGEkZQHh7t1mdg3hyz4XuM3d15jZjcAKd19qZm8C7gPKgXPN7AZ3P87d68zs3wghA3Cju9cNtYb8/HxmzZo16OP/8aZHOOWIcXznYl0JJCKS0jup3X0ZsKzfti8nrC8nNB8le+5twG2prK+/sng+Da1dI/mWIiIZa1R3Ug+3ccUx6ls7012GiEhGUEAkKIvHqG9RQIiIgAJiP+XxfOrVxCQiAigg9lMej7GnvYvuHt0sJyKigEhQHs/HHRrbdBYhIqKASFBeHANQM5OICAqI/ZTHQ0A06EomEREFRKK+gKjTlUwiIgqIRGXxMMSGbpYTEVFA7Gfc3j4InUGIiCggEsRjucRyc6hTQIiIKCASmVkYj6lFTUwiIgqIfjQek4hIoIDopyyer4AQEUEB8Trl8ZhulBMRQQHxOuXFMd0oJyKCAuJ1+kZ0Hcw81iIihzMFRD/l8Rg9vc6e9u50lyIiklYKiH40HpOISKCA6Ke8OAy3ofGYRCTbpTQgzGyJmb1kZhvN7Lok+wvM7K5o/9NmNjPaHjOzn5nZKjN7wczOTGWdicr2nkHoSiYRyW4pCwgzywV+BJwNzAUuM7O5/Q77GFDv7kcB3wW+EW3/OIC7zwfOAr5tZiNytjMurvGYREQgtWcQi4CN7r7J3TuBO4Hz+x1zPnB7tH4PsNjMjBAojwC4+y6gAViYwlr3Ko9r0iAREUhtQFQBWxMeV0fbkh7j7t1AI1ABvACcZ2Z5ZjYLOBmY1v8NzOwqM1thZitqamqGpeiSwjxyDOrVByEiWS5TO6lvIwTKCuB7wBNAT/+D3P0Wd1/o7gsrKyuH5Y1zciy6m1oBISLZLS+Fr72N/f/qnxptS3ZMtZnlAaVArYe71P6/voPM7Ang5RTWup+yeL46qUUk66XyDGI5MNvMZplZDLgUWNrvmKXAh6L1i4BH3N3NLG5mxQBmdhbQ7e5rU1jrfsrjMV3mKiJZL2VnEO7ebWbXAA8CucBt7r7GzG4EVrj7UuBW4A4z2wjUEUIEYALwoJn1Es4yrkxVncmUF8fYWtc6km8pIpJxUtnEhLsvA5b12/blhPV24ANJnrcZOCaVtR1IeTyfVdVqYhKR7JapndRpVR6PUdfaqQH7RCSrKSCSKC+O0dndS1vX6y6cEhHJGgqIJMrjYTwm3SwnItlMAZFE33hMullORLKZAiKJccUaj0lERAGRhJqYREQUEEmpiUlERAGRVFlR3xmEAkJEspcCIom83BzGFuZpPCYRyWoKiAGUF2s8JhHJbgqIAWjIbxHJdgqIAZRryG8RyXIKiAFoyG8RyXYKiAGUF8doUBOTiGQxBcQAyuP5tHT20NGtAftEJDspIAbQd7Oc+iFEJFspIAag8ZhEJNspIAZQ1jceU4vOIEQkOykgBlAe1xmEiGQ3BcQA1MQkItkupQFhZkvM7CUz22hm1yXZX2Bmd0X7nzazmdH2fDO73cxWmdk6M7s+lXUm09fEpE5qEclWKQsIM8sFfgScDcwFLjOzuf0O+xhQ7+5HAd8FvhFt/wBQ4O7zgZOBq/vCY6QU5OUSj+XqZjkRyVqpPINYBGx0903u3gncCZzf75jzgduj9XuAxWZmgAPFZpYHFAGdwJ4U1pqUxmMSkWyWyoCoArYmPK6OtiU9xt27gUagghAWLcAOYAvwLXevS2GtSZUXazwmEclemdpJvQjoAaYAs4DPmdkR/Q8ys6vMbIWZraipqRn2InQGISLZLJUBsQ2YlvB4arQt6TFRc1IpUAtcDvzB3bvcfRfwN2Bh/zdw91vcfaG7L6ysrBz2X6A8HtO0oyKStVIZEMuB2WY2y8xiwKXA0n7HLAU+FK1fBDzi7k5oVnoHgJkVA6cC61NYa1Ll8Xzq1cQkIlkqZQER9SlcAzwIrAPudvc1ZnajmZ0XHXYrUGFmG4F/Bvouhf0RMMbM1hCC5mfu/mKqah1IWTzGnvYuunt6R/qtRUTSLi+VL+7uy4Bl/bZ9OWG9nXBJa//nNSfbPtLK4/m4Q2NbFxVjCtJdjojIiMrUTuqMUL73bmo1M4lI9lFAHED53iG/1VEtItlHAXEAfQGhu6lFJBspIA6gvFjjMYlI9hpUQJjZZ81srAW3mtlKM3tXqotLNw35LSLZbLBnEB919z3Au4By4ErgppRVlSHisVxiuTnUKSBEJAsNNiAsWp4D3OHuaxK2HbbMLIzHpFnlRCQLDTYgnjWzhwgB8aCZlQBZcfeYxmMSkWw12BvlPgYsADa5e6uZjQM+krqyMkdZPF8BISJZabBnEKcBL7l7g5ldAXyRMDT3YW9ccUw3yolIVhpsQPwn0GpmJwCfA14Bfp6yqjJIWTymG+VEJCsNNiC6o1FWzwd+6O4/AkpSV1bm6BvRNfz6IiLZY7AB0WRm1xMub/29meUA+akrK3OUx2P09Dp72rvTXYqIyIgabEBcAnQQ7od4jTD5zzdTVlUG0XhMIpKtBhUQUSj8Aig1s/cC7e6eFX0QfcNtaDwmEck2gx1q42LgGcIcDRcDT5vZRaksLFPsO4PQlUwikl0Gex/EvwJviuaHxswqgT8C96SqsEyh8ZhEJFsNtg8ipy8cIrVDeO6opiG/RSRbDfYM4g9m9iDwq+jxJfSbSvRwVVKYR26OqYlJRLLOoALC3T9vZhcCp0ebbnH3+1JXVubIyTHKijTchohkn8GeQeDu9wL3prCWjKXxmEQkGx2wH8HMmsxsT5KfJjPb80YvbmZLzOwlM9toZtcl2V9gZndF+582s5nR9g+a2fMJP71mtuBgf8lDNa44Rr2G/BaRLHPAMwh3P+jhNMwsF/gRcBZQDSw3s6XuvjbhsI8B9e5+lJldCnwDuMTdf0G47wIzmw/c7+7PH2wth6osHmNrXWu63l5EJC1SeSXSImCju29y907gTsJYTonOB26P1u8BFptZ/4mILouemzblamISkSyUyoCoArYmPK6OtiU9xt27CUOIV/Q75hL2XT21HzO7ysxWmNmKmpqaYSk6mfJoyG8N2Cci2SSj72Uws1OAVndfnWy/u9/i7gvdfWFlZWXK6iiPx+js7qWtqydl7yEikmlSGRDbgGkJj6dG25IeY2Z5QCnhJrw+lzLA2cNIKo+H8Zg0cZCIZJNUBsRyYLaZzTKzGOHLfmm/Y5YCH4rWLwIeieadIBpS/GLS3P8ACcNt6G5qEckig74PYqjcvdvMrgEeBHKB29x9jZndCKxw96XArcAdZrYRqCOESJ8zgK3uvilVNQ5WebHGYxKR7JOygABw92X0G5LD3b+csN5OGCE22XMfA05NZX2DpSYmEclGGd1JnSnUxCQi2UgBMQilRX1nEAoIEckeCohByMvNYWxhnkZ0FZGsooAYpHHFMc0JISJZRQExSGXxmJqYRCSrKCAGqTyeryYmEckqCohBKlcTk4hkGQXEIB0xvpjtjW2s2d6Y7lJEREaEAmKQrjxtJqVF+Xx92XqN6ioiWUEBMUilRfl85h2zeXzjbv78cuqGFhcRyRQKiCG44tQZzKiI8/Vl6+np1VmEiBzeFBBDEMvL4dolx/LSzibueXbrGz9BRGQUU0AM0dnzJnHS9DK+/dDLtHZ2p7scEZGUUUAMkZnxr++Zw66mDn76l7+nuxwRkZRRQByEk2eM4+x5k7j5L6+wq6k93eWIiKSEAqKrDVbdA/WvDulp1y45ls7uXr778IYUFSYikl4KiLZ6uPcf4bn/GdLTZo4v5opTZ3DX8i1s2NmUouJERNJHATF2ChzxNnjxLhjiDXCfWTyb4lgeX39gfYqKExFJHwUEwPGXQsOrsOWpIT1tXHGMT7/jKB5Zv4snNu5OUXEiIumhgACYcy7kx+HFO4f81A+/eSZVZUX8+7J19OrmORE5jKQ0IMxsiZm9ZGYbzey6JPsLzOyuaP/TZjYzYd/xZvakma0xs1VmVpiyQgvGhJBYfR90De2qpML8XD7/7mNYs30Pv3lhW4oKFBEZeSkLCDPLBX4EnA3MBS4zs7n9DvsYUO/uRwHfBb4RPTcP+B/gE+5+HHAmkNrJGI6/BDoa4eU/DPmp550whXlVY/nmH16ivasnBcWJiIy8VJ5BLAI2uvsmd+8E7gTO73fM+cDt0fo9wGIzM+BdwIvu/gKAu9e6e2q/eY84E8ZMCp3VQ5STY3zhnDlsb2znZ3/bPNyViYikRSoDogpIHLCoOtqW9Bh37wYagQrgaMDN7EEzW2lm/5LsDczsKjNbYWYramoOcYTVnFw4/gOw4SFoqR3y09985HgWHzuBHz+6kdrmjkOrRUQkA2RqJ3Ue8Bbgg9HyAjNb3P8gd7/F3Re6+8LKyspDf9fjL4Xebljz64N6+vXnHEtrVw9X3fEsjW2anlRERrdUBsQ2YFrC46nRtqTHRP0OpUAt4WzjL+6+291bgWXASSmsNZg0DybOgxd+dVBPP2pCCT+47ERerG7g8p8+pTMJERnVUhkQy4HZZjbLzGLApcDSfscsBT4UrV8EPOJhurYHgflmFo+C423A2hTWus/xl8C2Z2H3wQ2hcc78ydzyDwvZuKuZS255itcaNVaTiIxOKQuIqE/hGsKX/TrgbndfY2Y3mtl50WG3AhVmthH4Z+C66Ln1wHcIIfM8sNLdf5+qWvcz/wNgOQfVWd3n7cdM4PaPLuK1xnY+cPMTbK1rHcYCRURGhh0u8ysvXLjQV6xYMTwvdscFULsRPvMC5Bx8hr6wtYF/uO0ZCvNz+MU/nsJRE0qGpz4RkWFiZs+6+8Jk+zK1kzq9jr8UGrbAlicP6WVOmFbGXVefSk8vXHzzU6ze1jhMBYqIpJ4CIpk574X84oMaeqO/YyeN5X8/cRpF+blc9tOnWLG5bhgKFBFJPQVEMrFimHserPnNkIfeSGbW+GLu/sRpjB9TwJW3PsPjGzSwn4hkPgXEQPYOvfHAsLxcVVkRd199GjMq4nz0v5fz4JrXhuV1RURSRQExkFlnQMlkeOHgr2bqr7KkgDuvOpU5U8Zy9R3P8rm7X9C9EiKSsRQQA8nJDZe8bnwYWoavSagsHuPOj5/KJ888kt88v413fPvP/PLpLRoqXEQyjgLiQE64LAy9sfreYX3Zolgu1y45lgc++1aOnVTCF+5bxYU/eYI123WVk4hkDgXEgUycC5PmwwuHfjVTMrMnlnDnVafy7Q+cwJbaVs79wePc+Nu1NHd0p+T9RESGQgHxRo6/FLavhJqXU/LyZsaFJ0/lkc+dyaWLpvOzJ/7O4m8/xu9f3MHhchOjiIxOCog3Mv+iaOiN1JxF9CmN5/O1C+bz60++mYriAj79y5V86GfLdXOdiKSNAuKNlEyCI98BL94Nvb0pf7sTp5ez9JrT+cq5c1n5aj3v/cHjvP/Hf+P+57bR0a3Z6kRk5CggBuP4S6FxK2x5YkTeLi83h4+cPou/XfcOvvTeudS3dvFPdz3P6Tc9wrcefIntDW0jUoeIZDcN1jcYna3wrdlw3AVw/g9T8x4H0NvrPL5xNz9/cjN/Wr+LHDPOmjORfzhtBqcdWUGYpVVEZOgONFhf3kgXMyrF4nDc+8JEQhPmwqmfhBH8Us7JMc44upIzjq5ka10rv3h6C3ct38If1rzGURPGcOWpM7jgpCrGFuaPWE0icvjTGcRgtTXA/Z+El5bBse+F838ERWWpe7830N7Vw+9e3MEdT27mhepGivJzOe+EKXzw1OkcPzV9dYnI6HKgMwgFxFC4w5M/hD9+FUqnwgduhykLUvueg/BidQO/fHoLv3l+O21dPcyvKuWDp0zn3BOmUFygk0QRGZgCYrhtfQb+98PQUgNLvg4LPzaiTU4D2dPexf3PbeMXT23hpZ1NjCnI44ITq/jgqdM5dtLYdJcnIhlIAZEKLbVw39VhrKbj3g/nfR8KMmPGOHfn2Vfr+cXTW/j9qh10dvdy8oxy3n9SFe+aO4nKkoJ0lygiGUIBkSq9vfC378Ij/xfGHRGanCbNG9ka3kB9Syf3rqzml89sYVNNCzkGb5o5jnPmT2bJvElMHFuY7hJFJI0UEKm2+XG452PQ3gDnfBNOvDIjmpwSuTsv7Wxi2arXeGDVDjbsagZg4YxylsybxNnzJ1NVVpTmKkVkpKUtIMxsCfD/gFzgv9z9pn77C4CfAycDtcAl7r7ZzGYC64CXokOfcvdPHOi90hoQAM274Ncfh02PwTHnwJKboHxG+up5Axt3NfHAqtdYtvo11u3YA4Q5tM+aM4ETppUxv6qUsngszVWKSKqlJSDMLBd4GTgLqAaWA5e5+9qEYz4FHO/unzCzS4EL3P2SKCB+5+6Dbq9Je0AA9PaEq5weuylc8XTG5+DNn4G8zG7z37y7hQdWv8YDq3fwYvW+sZ+mlhcxv6qUeVWlzI9+yosVGiKHk3QFxGnAV9393dHj6wHc/esJxzwYHfOkmeUBrwGVwAxGY0D0aayGP1wP65ZCxVGh2enId6S7qkFpbO1i9fZGXqxuZPW2RlZta2RLXeve/VVlITTmTy3lhKnhTKM0rhv0REardN1JXQVsTXhcDZwy0DHu3m1mjUBFtG+WmT0H7AG+6O5/7f8GZnYVcBXA9OnTh7f6Q1E6FS65Azb+EZZ9Hu64AOa+D979NSitSnd1B1Qaz+f0o8Zz+lHj927rC41VUWCsqm7kDwlzas+oiDO/qpTjp5Zy/NQyjpsylhLd1S0y6mXqXVQ7gOnuXmtmJwP3m9lx7r4n8SB3vwW4BcIZRBrqPLCj3gmffBKe+D789duw4WE481o49VOQO3q+QJOFRkNrJ6u37eHFbQ2sqm7kuS0N/O7FHUDonz9ifDHzq0o5bkopx00Zy9wpY9WnITLKpDIgtgHTEh5PjbYlO6Y6amIqBWo9tHt1ALj7s2b2CnA0kCFtSEOQXwhv+xc4/mJ44Dp4+Mvw/C9DJ/YRZ2bc1U6DVRaP8ZbZ43nL7H2hsbu5Y+8ZxovVjTy1qY77n9++d39VWRFzJo/luClj94ZGVVmRBhsUyVCp7IPII3RSLyYEwXLgcndfk3DMp4H5CZ3U73f3i82sEqhz9x4zOwL4a3Rc3UDvl1F9EAfy0gPwwL9AwxaonAMLPwonXAKFpemuLCVqmztYu2MPa7bvYe32PazZ3sim3S30/W9XWpS/NzD6zjaOqBxDbo5CQ2QkpPMy13OA7xEuc73N3f/dzG4EVrj7UjMrBO4ATgTqgEvdfZOZXQjcCHQBvcBX3P23B3qvURMQAF1tsOoeWHErbH8O8uNh5sWkaZ0AABLgSURBVLqFH4UpJ6a7upRr7exm/WtNUWDsYe32Rta91kRnd5iQqSg/l2Mnl+wXGkdPLKEwPzfNlYscfnSjXCbbthKe/VkIjK7WEBALPwrzLoRYcbqrGzFdPb28UtPMmm0hNNZsb2Tt9j00dXQDoSVuYkkhVeVFTI1+qsrie9enlBUpQEQOggJiNGhrCNOarrgNatZBQSmccCks/AhMmJPu6tLC3dla18bq7Y28vLOJ6vo2ttW3Ud3Qyo6Gdrp79/9/t7KkIAqMONP6luOKmFYeZ0pZEbE8TaAo0p8CYjRxhy1PhaBYez/0dMK0U+CkD4UZ7WLxdFeYEXp6nZ172qmub6O6vpVt9W1srW9lW0MbW+va2N7Qtl+AmMGksYVMLS9i+rhijp1UwpzJYzl2cgnjx2T2jYwiqaSAGK1aasMsds/+N9RuCGcVx38ATv4wTJqf7uoyWndPLzubOtha10p1fdu+ZX0rf9/dQk1Tx95jK0sKOHZSCXOjwJgzeSxHVo4hP1dnHHL4U0CMdu6w5ckQFGvuh54OmHJSCIp5F0LBmHRXOOrUNnew/rUm1u3Yw7odTax/bQ8bdjbT2RM6yvNzjVnjizli/BiOqCzmiMqwPHL8GN05LocVBcThpLUu9FWsvB12rYXYmDBf9ux3wxFvO2wvlx0JXT29/H13C+t27GHtjj28squFTbub2VLbul9zVUVxLITG+DHMHF/MxLEFTCgp3LscW5Sneztk1FBAHI7coXp5OKtY91vo2AOWC9MWwVGL4cjFMHkB5KiZ5FB19fSyta6VV2pa2FTTzKaaEBybalqobel83fEFeTlMiMJiQkkBE8cWUlEco7w4tnc5rjhGeTxGeTyfPDVlSRopIA53PV0hLDb+KYz/tOP5sD0+PgwSeNTisBwzIb11Hoaa2rvY1dTBrj0d7Gpqp6apg5172vfbtmtPx97LdZMpLcqPAiN/b3CM6wuSxPXocUlhHjm6kVCGiQIi2zTXwCuPwCt/CqHRujtsL50G42dDxeyw7FsfO2XUDvkxWnR099DQ2kVdS+fen/rWaNnSSV1rF3UtHdS1dIXHLZ17+0P6y82xKERCoFQUF1BenM+44gIq+oIk4acsnk9Bnu4RkeQUENmstxdeewFeeRR2rYPdL0PtRuhs3ndMbAxUHBnComwa5OSD5UBObsIyd/9lbEzo7ygcGy2jn1iJmrWGgbvT2tmzf5C0dlLbvO9x33ptFCgNrV0Dvt6YgrwQIvF9ZyblxTEqSwqYXFrI5NIiJpcWMmFsgcIky6RruG/JBDk54e7sxCE83KFpB+zesC8wdm+Arc+Eey96B24OeWMWQqOgFEomhns4ZrwZpp8G8XGH/OtkCzOjuCCP4oI8po0b3L0v3T29NLR17Q2PhtZO6lqjM5SWrr3BUtfSycZdzdS1dNLa2fO61xk/Jsak0kImjQ2hMXFsAWMK8ogX5FEcyyMeyyUey6W4IG//ZUxNX4cbnUFIcr294D1hljzvAe+N1ntDgHQ2Q3sjtO+Jlgk/HdG2+ldh27PhslwIgxPOePO+wBjJuTG62mDDQ2FIkx0vwLgjoPJYqDxm3zILA6ypvYude9rZ0Rh+Xtu7bNu7rbFt4DOTRDkW+lPK4qFZqzweo2y/x2G9YkyM8WNCc1hZPKaBGdNMTUySPt0dYbypLU/Aq0/AlqehsynsK5sBM06H6afA1EXhi3o4m6d6usIc4avugfW/D+9bPAGmnxpG0939chj/qk/xhBAUE+aE5YzTQ01Z3j/T0d1Da0cPLZ3dtHb20NLRb9nZTUtHN03t3dS3hqauhtauhPVOWpKcqUAIlb5+lIoxMSqi4CguyCU3J4e8HCM3+snbb5lDfq4xvqRg75Vi4+IxncEcBAWEZI6ebti5OgqLKDRaa8O+grFQdXK4VHfqIpi6EIrKhvb6vb3hpsLV94SbCtvqQnPX3HNh3kUw862Qm7fv2MatUPMS1Kzff9kXYhWzYe55MOc8mHzC0MLCPQTR9uf2hc6h6O6A5/4nnMXNOgPGHz1qwquzu5eGthAYtc2d1LZ0hGVzBzXRsrYlWjZ30trVQ0/v0L6b8nKMyigwKhPuSykpzCM/18jLDYGTn5tDXm4ImrycsB7Ly2H8mAIqxxRQFs/PqvtYFBCSudyhblPo/6h+BrYuh11rwpcgwPhjQmBMmAt4OCvo7QrLvevdYcyq7vbQGd+0PQyhfszZIRSOWgx5Qxhvqe+LfcNDYV7xzX8LzWxl00NQzD0fqha+/mynuyM0X219BrY+HZbN0dSslgMnXglv/9fQNzPUz2jdb+HhL0H95n3bSyaHoJh1Bsx6W7jA4DDi7vQ6dPf20tPrdPc6PT1Ojzs9vU5HVy81zR3sii4r3nt5cdO+bXVJ7lN5I/m5FsKiJARGZUn4GT+mgHHFMcYW5TO2MC9a5jO2KG9Ud+wrIGR06WgKzVJ9oVG9HNrqX39cTh7kxsJVV7nR+uQFYW6No5cM3xAkLbXw0rIQFq88GkKpZDLMOTcExWsvhhq3PxeCCqB8ZjgLmrYo1LTm1/DMT0ONp38W3nzN4IZz3/EC/OEL8OrjoQ9nydegfBb8/c/w97+En5aacOy4I0JQzDojNKMVlEBuQZjeNov+Ik7U2d1La2c3XT1Od28v3T1OV08v3b1Od7Stq8fp6OqhtqWTmqYOapo7wrLvp7mD2uYODnRCU5CXQ2lR/t7weF0/THSfS3k8RmlRPuXFMYryc3F3nPA3gOMQvUffthwLU/6mMoAUEDK6uYchRnJywxdsbn4Ih3R86bU3wssPwtrfhHtMutvCl/CUE2Ham8JVW1MXJT9LqH0F/nRDeO6YSfCOL8KCy8Pv1V/TTnjk30KTUnxcOPM46UP7msf6uIchVzb9OYTG5r/tax5LlBvbFxZ5BdHjWLSev+9xX+gmbisqC1PmVp08PJ/hKNTT69GlxJ3sae9iT1t3tOxiT3t3tAzbG9sG1/8yFCWFeVSOKdjXwb93WUDlmBhVZXHmTz24YXYUECKp0NkSmsfGHz20JqwtT8ND/xrOOiYcB+/6t9AMBtDVDk/9GP767dBkdcrVcMbnB98X09MdzmS2Pxea3Ho6oLsznNn0/XR37Fv2difs60pYT9jeUhM686sWhnrmvg/yYkP/vIab+8j8kdDdGUJ9+X+FJsOyGeEMsXxGtD4rrMcrktbT0d1DY2sX9VFg9C07ohkUzcCiFWPfSxhGrzv1LeFel93NHeyO+mh2N3dQn3Dfy4JpZdz/6dMP6tdTQIhkGvdwz8nDX4GGV8PYWXPOhce/E/o/jnlPCI6KI9NdabiU+YU74Zmbwz0zYybCyR8Jk1mVTDr413UPlx93Nkc/LdARLdsbwlljW11YttbuW+9bdneEprTZZ8Hsdw3/FWfNu2DFz8LUwM07YdyRMPn48N+n/tV9IxT0iY2JAmNG6K/q/1NYNqz1dfX0Ut/Sye7mTnrdmVelM4gBKSBkVOruCH+Z/vk/wpfihONCP8MRZ6a7stfr7YVNj8DTN4cO/Jz8MJLwKZ8IV5wl6myBur9D/d/3XzZWhz6mzpbobv5BfP8UlkLRuNDUFq/Yt46FZrWdq8NxpdPgqHeGsJh1xsH3QW1bGX7HNb8OZ1BHvTP8jkcu3v/ChI7mEO71r0bLzfvWG7bsP1oBhKv0EgMjNiacmXW1Qme07Grbt62rLZzVVRwJE4+DifPCz/jZoQlwmCggRDJdax3sXBNuIkzWJ5Fpal8Jne7P/yLcGDnlpNDU1hcELbv2P76wDMbNiv6SLg1fjrHi/ZcFCet9oVBU/vp+l/4at4VBKjc8FO576WwOfSczTg9nF5Pm93uvaD3xS7anKzQjPX1zuDAiNgYWfBAWfTx8IQ+Ve7iwouFVaNgaAmO/n1dDCOQXh1ki84vClXf5ietF4eq33RvC5de9UZNSTn44W5o0LwqO42DifBhTOfQ6UUCISKp0NIXmpxW3hQ788lkwbma0nLVvWVQ+MvV0d4b7YDY+DBseDl+sA8mN7QuLzpbQdDXuCFh0dbh4oHBsamsdSh9KT1cIip1rYOeqaLkmDJkDIQQ/8fhBlZG2gDCzJcD/A3KB/3L3m/rtLwB+DpwM1AKXuPvmhP3TgbXAV939Wwd6LwWEiLxO31/sfU1anS391lvD0nvDnO9HvXN0DTbZsjsERU8XzH7nQb1EWgbrM7Nc4EfAWUA1sNzMlrr72oTDPgbUu/tRZnYp8A3gkoT93wEeSFWNInKY62vvP1wVjw8zSaZIKqNyEbDR3Te5eydwJ3B+v2POB26P1u8BFlt0j7uZvQ/4O7AmhTWKiMgAUhkQVcDWhMfV0bakx7h7N9AIVJjZGOBa4IYDvYGZXWVmK8xsRU1NzbAVLiIiqQ2IQ/FV4Lvu3nygg9z9Fndf6O4LKysPrgdfRESSS+WEQduAxNHDpkbbkh1TbWZ5QCmhs/oU4CIz+w+gDOg1s3Z3/2EK6xURkQSpDIjlwGwzm0UIgkuBy/sdsxT4EPAkcBHwiIfLqt7ad4CZfRVoVjiIiIyslAWEu3eb2TXAg4TLXG9z9zVmdiOwwt2XArcCd5jZRqCOECIiIpIBdKOciEgWO9B9EJnaSS0iIml22JxBmFkN8OohvMR4YPcbHpWZRnPtMLrrH821w+iufzTXDplT/wx3T3oZ6GETEIfKzFYMdJqV6UZz7TC66x/NtcPorn801w6jo341MYmISFIKCBERSUoBsc8t6S7gEIzm2mF01z+aa4fRXf9orh1GQf3qgxARkaR0BiEiIkkpIEREJKmsDwgzW2JmL5nZRjO7Lt31DJWZbTazVWb2vJll/K3kZnabme0ys9UJ28aZ2cNmtiFajtD8lEMzQO1fNbNt0ef/vJmdk84aB2Jm08zsUTNba2ZrzOyz0fbR8tkPVH/Gf/5mVmhmz5jZC1HtN0TbZ5nZ09F3z11mFkt3rf1ldR9ENOvdyyTMegdc1m/Wu4xmZpuBhe6eCTfcvCEzOwNoBn7u7vOibf8B1Ln7TVFIl7v7temsM5kBav8qYTDJA06Jm25mNhmY7O4rzawEeBZ4H/BhRsdnP1D9F5Phn380CVqxuzebWT7wOPBZ4J+BX7v7nWb2E+AFd//PdNbaX7afQQxm1jsZRu7+F8LAjIkSZxa8nfAPP+MMUPuo4O473H1ltN4ErCNM2DVaPvuB6s94HvTNbZMf/TjwDsJMmpChn322B8RgZr3LdA48ZGbPmtlV6S7mIE109x3R+mvAxHQWcxCuMbMXoyaojGyiSWRmM4ETgacZhZ99v/phFHz+ZpZrZs8Du4CHgVeAhmgmTcjQ755sD4jDwVvc/STgbODTUTPIqBXNBzKa2j3/EzgSWADsAL6d3nIOLJrO917gn9x9T+K+0fDZJ6l/VHz+7t7j7gsIE6ctAo5Nc0mDku0BMZhZ7zKau2+LlruA+wj/8402O6M25r625l1prmfQ3H1n9I+/F/gpGfz5R+3f9wK/cPdfR5tHzWefrP7R9PkDuHsD8ChwGlAWzaQJGfrdk+0BsXfWu+gKgksJs9yNCmZWHHXYYWbFwLuA1Qd+Vkbqm1mQaPmbNNYyJH1frpELyNDPP+oovRVY5+7fSdg1Kj77geofDZ+/mVWaWVm0XkS4KGYdISguig7LyM8+q69iAogui/se+2a9+/c0lzRoZnYE4awBwuyAv8z0+s3sV8CZhKGOdwJfAe4H7gamE4Zsv9jdM64zeIDazyQ0bziwGbg6oU0/Y5jZW4C/AquA3mjzFwjt+KPhsx+o/svI8M/fzI4ndELnEv4ov9vdb4z+/d4JjAOeA65w9470Vfp6WR8QIiKSXLY3MYmIyAAUECIikpQCQkREklJAiIhIUgoIERFJSgEhkgHM7Ewz+1266xBJpIAQEZGkFBAiQ2BmV0Rj+z9vZjdHg7A1m9l3o7H+/2RmldGxC8zsqWggufv6BpIzs6PM7I/R/AArzezI6OXHmNk9ZrbezH4R3T0skjYKCJFBMrM5wCXA6dHAaz3AB4FiYIW7Hwf8mXCHNcDPgWvd/XjCHcB9238B/MjdTwDeTBhkDsIIpf8EzAWOAE5P+S8lcgB5b3yIiEQWAycDy6M/7osIg9v1AndFx/wP8GszKwXK3P3P0fbbgf+Nxs6qcvf7ANy9HSB6vWfcvTp6/DwwkzC5jEhaKCBEBs+A2939+v02mn2p33EHO35N4jg8Pejfp6SZmphEBu9PwEVmNgH2zuc8g/DvqG9UzsuBx929Eag3s7dG268E/hzNhlZtZu+LXqPAzOIj+luIDJL+QhEZJHdfa2ZfJMzglwN0AZ8GWoBF0b5dhH4KCEM4/yQKgE3AR6LtVwI3m9mN0Wt8YAR/DZFB02iuIofIzJrdfUy66xAZbmpiEhGRpHQGISIiSekMQkREklJAiIhIUgoIERFJSgEhIiJJKSBERCSp/x+ibHuB+PI2pAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6Mai95nuzJzF"
      },
      "source": [
        "# Archive scan tests\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OuBQBu5Qzz4o",
        "outputId": "a12f3953-c458-49e1-cf54-673db590ca4b"
      },
      "source": [
        "!ls -l WEIGHTS/"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 7884\n",
            "-rw-r--r-- 1 root root  748952 Aug 13 12:22 weights_model_cnn_spec.hd5\n",
            "-rw-r--r-- 1 root root 7032472 Aug 13 12:22 w_gpd_scsn_2000_2017.h5\n",
            "-rw-r--r-- 1 root root  289936 Aug 13 12:22 w_model_performer_with_spec.hd5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "daKR-TJNzcTh"
      },
      "source": [
        "### regular model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xDjaN84BdKR4",
        "outputId": "cc186cbf-a356-470f-85dc-0852fabed753"
      },
      "source": [
        "%%time\n",
        "\n",
        "!rm predictions.txt\n",
        "!python archive_scan.py test/nysh_archives.txt --threshold \"p: 0.9997, s: 0.9995\" --batch-size 200000 --trace-size 6000 --time --print-precision 10\n",
        "!cat predictions.txt"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Group 1 out of 1 [=====================================>..] - Batch: 2021-04-01T23:20:03.660000Z - 2021-04-02T00:00:00.210000Z Time: 3.31063 seconds\n",
            "Total model prediction time: 3.39315 seconds\n",
            "NYSH P 0.9997605681 01.04.2021 12:35:55.66\n",
            "NYSH S 0.9996142387 01.04.2021 12:36:04.46\n",
            "NYSH P 0.9997449517 01.04.2021 19:19:18.86\n",
            "CPU times: user 305 ms, sys: 93.9 ms, total: 399 ms\n",
            "Wall time: 42 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fctdFdmz15tp"
      },
      "source": [
        "### CNN model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EfgC9H2kgAFQ",
        "outputId": "10fa11e2-e1a6-4214-dc0a-430a162c9d69"
      },
      "source": [
        "%%time\n",
        "\n",
        "!rm predictions.txt\n",
        "!python archive_scan.py test/nysh_archives.txt --cnn --threshold \"p: 0.9999, s: 0.9995\" --batch-size 200000 --trace-size 6000 --time --print-precision 10\n",
        "!cat predictions.txt"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Group 1 out of 1 [=====================================>..] - Batch: 2021-04-01T23:20:03.660000Z - 2021-04-02T00:00:00.210000Z Time: 16.4322 seconds\n",
            "Total model prediction time: 16.6955 seconds\n",
            "NYSH P 0.9999938011 01.04.2021 12:35:55.26\n",
            "NYSH S 0.9999948740 01.04.2021 12:36:04.46\n",
            "CPU times: user 363 ms, sys: 130 ms, total: 493 ms\n",
            "Wall time: 54.6 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KQdDAnxqyTk-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b6c2678b-31b3-4afe-f499-88ca5eeaafb7"
      },
      "source": [
        "%%time\n",
        "\n",
        "!rm predictions.txt\n",
        "!python archive_scan.py test/nysh_archives.txt --gpd --threshold \"p: 0.9999, s: 0.9995\" --batch-size 200000 --trace-size 6000 --time --print-precision 10\n",
        "!cat predictions.txt"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "rm: cannot remove 'predictions.txt': No such file or directory\n",
            "Group 1 out of 1 [=====================================>..] - Batch: 2021-04-01T23:20:03.660000Z - 2021-04-02T00:00:00.210000Z Time: 5.92907 seconds\n",
            "Total model prediction time: 6.32017 seconds\n",
            "NYSH S 0.9996261596 01.04.2021 02:57:06.06\n",
            "NYSH S 0.9997906088 01.04.2021 12:36:04.46\n",
            "CPU times: user 321 ms, sys: 113 ms, total: 434 ms\n",
            "Wall time: 47.3 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CQ55KZ0evqyK"
      },
      "source": [
        "# Train Seismo-Performer on Sakhalin Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EqCitwVrAHok"
      },
      "source": [
        "### model spec"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VNNTsHxVvwNP",
        "outputId": "18d7c189-15f5-4080-f406-7c230b6b9677"
      },
      "source": [
        "model_with_spec_sakh = seismo_performer_with_spec(\n",
        "                                        maxlen=400,\n",
        "                                        nfft=64,\n",
        "                                        hop_length=16,\n",
        "                                        patch_size_1=22,\n",
        "                                        patch_size_2=3,\n",
        "                                        num_channels=3,\n",
        "                                        num_patches=11,\n",
        "                                        d_model=48,\n",
        "                                        num_heads=2,\n",
        "                                        ff_dim_factor=2,\n",
        "                                        layers_depth=2,\n",
        "                                        num_classes=3,\n",
        "                                        drop_out_rate=0.1)\n",
        "\n",
        "LR = 0.0001\n",
        "model_with_spec_sakh.compile(\n",
        "    optimizer=keras.optimizers.Adam(learning_rate=LR),\n",
        "    loss=keras.losses.SparseCategoricalCrossentropy(),\n",
        "    metrics=[keras.metrics.SparseCategoricalAccuracy()],)\n",
        "\n",
        "model_with_spec_sakh.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 400, 3)]          0         \n",
            "_________________________________________________________________\n",
            "stft (STFT)                  (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude (Magnitude)        (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude_to_decibel (Magnit (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "max_abs_scaler (MaxABSScaler (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "rearrange3d (Rearrange3d)    (None, 11, 198)           0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 11, 48)            9552      \n",
            "_________________________________________________________________\n",
            "cls_token (ClsToken)         (None, 12, 48)            48        \n",
            "_________________________________________________________________\n",
            "pos_embeding2 (PosEmbeding2) (None, 12, 48)            576       \n",
            "_________________________________________________________________\n",
            "performer_block (PerformerBl (None, 12, 48)            18672     \n",
            "_________________________________________________________________\n",
            "performer_block_1 (Performer (None, 12, 48)            18672     \n",
            "_________________________________________________________________\n",
            "lambda (Lambda)              (None, 48)                0         \n",
            "_________________________________________________________________\n",
            "layer_normalization_4 (Layer (None, 48)                96        \n",
            "_________________________________________________________________\n",
            "dropout_4 (Dropout)          (None, 48)                0         \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 96)                4704      \n",
            "_________________________________________________________________\n",
            "dropout_5 (Dropout)          (None, 96)                0         \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 48)                4656      \n",
            "_________________________________________________________________\n",
            "dropout_6 (Dropout)          (None, 48)                0         \n",
            "_________________________________________________________________\n",
            "dense_7 (Dense)              (None, 3)                 147       \n",
            "=================================================================\n",
            "Total params: 57,123\n",
            "Trainable params: 57,123\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hyvPVSTdACD_"
      },
      "source": [
        "### Train the model with different random seeds"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_FyphDmOv39X",
        "outputId": "43fe2a67-24f4-47ee-f7b9-47622a32e122"
      },
      "source": [
        "%%time\n",
        "\n",
        "interations = 10\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 5, restore_best_weights = True, verbose = 1),\n",
        "    keras.callbacks.ModelCheckpoint('w_performer_epoch_hpa{epoch}.h5', save_weights_only=True),\n",
        "]\n",
        "\n",
        "sparse_categorical_accuracy = []\n",
        "loss = []\n",
        "val_loss = []\n",
        "val_sparse_categorical_accuracy = []\n",
        "\n",
        "cal_top_1 = []\n",
        "sakh_top_1 = []\n",
        "dag_top_1 = []\n",
        "\n",
        "# load California data for test\n",
        "X_train, X_test = h5_tts('/content/scsn_ps_2000_2017_shuf.hdf5',\\\n",
        "                             batch_size = 480,\\\n",
        "                             test_size = 0.99, random_state = 0, shuffle = False)\n",
        "\n",
        "\n",
        "for i in range(interations):\n",
        "  print(\"Load dataset with random state {}\".format(i))\n",
        "  X_train_sakh, X_test_sakh, y_train_sakh, y_test_sakh =\\\n",
        "      load_test_train_data('/content/drive/MyDrive/DATA/ML_DATASETS_SAKH_DAG/SAKH-2014_2021_global_norm.h5', 0.2, random_state=i)   \n",
        "\n",
        "  print(\"compile the model\")\n",
        "  model_with_spec_sakh = seismo_performer_with_spec(\n",
        "                                        maxlen=400,\n",
        "                                        nfft=64,\n",
        "                                        hop_length=16,\n",
        "                                        patch_size_1=22,\n",
        "                                        patch_size_2=3,\n",
        "                                        num_channels=3,\n",
        "                                        num_patches=11,\n",
        "                                        d_model=48,\n",
        "                                        num_heads=2,\n",
        "                                        ff_dim_factor=2,\n",
        "                                        layers_depth=2,\n",
        "                                        num_classes=3,\n",
        "                                        drop_out_rate=0.1)\n",
        "\n",
        "\n",
        "\n",
        "  LR = 0.0001\n",
        "  model_with_spec_sakh.compile(\n",
        "      optimizer=keras.optimizers.Adam(learning_rate=LR),\n",
        "      loss=keras.losses.SparseCategoricalCrossentropy(),\n",
        "      metrics=[keras.metrics.SparseCategoricalAccuracy()],)\n",
        "\n",
        "  print(\"Fit model on training data step {}\".format(i+1))\n",
        "  history = model_with_spec_sakh.fit(\n",
        "    X_train_sakh,\n",
        "    y_train_sakh,\n",
        "    #shuffle=False,\n",
        "    epochs=400,\n",
        "    validation_data=(X_test_sakh, y_test_sakh),\n",
        "    callbacks = callbacks\n",
        "    )\n",
        "  \n",
        "  #save convergance curve \n",
        "  sparse_categorical_accuracy.append(history.history['sparse_categorical_accuracy'])\n",
        "  loss.append(history.history['loss'])\n",
        "  val_sparse_categorical_accuracy.append(history.history['val_sparse_categorical_accuracy'])\n",
        "  val_loss.append(history.history['val_loss'])\n",
        "\n",
        "  print(\"Evaluate transformer based model on SAKHALIN test data\")\n",
        "  sakh_top_1.append(model_with_spec_sakh.evaluate(X_test_sakh, y_test_sakh, batch_size=128)[1])\n",
        "\n",
        "  print(\"Evaluate transformer based model on DAGESTAN test data\")\n",
        "  dag_top_1.append(model_with_spec_sakh.evaluate(X_test_dag, y_test_dag, batch_size=128)[1])\n",
        "  \n",
        "  print(\"Evaluate transformer based model on CALIFORNIA test data\")\n",
        "  cal_top_1.append(model_with_spec_sakh.evaluate(X_test)[1])\n",
        "\n",
        "  model_with_spec_sakh.save_weights('/content/drive/MyDrive/MODELS/w_model_performer_with_spec_sakh_seeds.hd5', save_format='h5')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Load dataset with random state 0\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.28992879 0.39572737 0.31434385]%\n",
            "train P, S and noise labels is [0.31484544 0.37641521 0.30873935]%\n",
            "compile the model\n",
            "Fit model on training data step 1\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 7s 16ms/step - loss: 0.6210 - sparse_categorical_accuracy: 0.7481 - val_loss: 0.3426 - val_sparse_categorical_accuracy: 0.8825\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.3252 - sparse_categorical_accuracy: 0.8878 - val_loss: 0.2741 - val_sparse_categorical_accuracy: 0.9079\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2657 - sparse_categorical_accuracy: 0.9087 - val_loss: 0.2456 - val_sparse_categorical_accuracy: 0.9176\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2311 - sparse_categorical_accuracy: 0.9225 - val_loss: 0.2495 - val_sparse_categorical_accuracy: 0.9237\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2086 - sparse_categorical_accuracy: 0.9330 - val_loss: 0.2297 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1961 - sparse_categorical_accuracy: 0.9379 - val_loss: 0.2315 - val_sparse_categorical_accuracy: 0.9278\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1842 - sparse_categorical_accuracy: 0.9412 - val_loss: 0.2241 - val_sparse_categorical_accuracy: 0.9237\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1738 - sparse_categorical_accuracy: 0.9439 - val_loss: 0.2135 - val_sparse_categorical_accuracy: 0.9339\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1676 - sparse_categorical_accuracy: 0.9487 - val_loss: 0.2148 - val_sparse_categorical_accuracy: 0.9329\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1582 - sparse_categorical_accuracy: 0.9504 - val_loss: 0.2165 - val_sparse_categorical_accuracy: 0.9329\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1524 - sparse_categorical_accuracy: 0.9513 - val_loss: 0.2153 - val_sparse_categorical_accuracy: 0.9359\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1445 - sparse_categorical_accuracy: 0.9534 - val_loss: 0.2096 - val_sparse_categorical_accuracy: 0.9359\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1375 - sparse_categorical_accuracy: 0.9557 - val_loss: 0.2068 - val_sparse_categorical_accuracy: 0.9364\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1306 - sparse_categorical_accuracy: 0.9584 - val_loss: 0.2133 - val_sparse_categorical_accuracy: 0.9400\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1270 - sparse_categorical_accuracy: 0.9611 - val_loss: 0.2088 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1250 - sparse_categorical_accuracy: 0.9618 - val_loss: 0.2226 - val_sparse_categorical_accuracy: 0.9395\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1132 - sparse_categorical_accuracy: 0.9651 - val_loss: 0.2129 - val_sparse_categorical_accuracy: 0.9400\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1136 - sparse_categorical_accuracy: 0.9654 - val_loss: 0.2207 - val_sparse_categorical_accuracy: 0.9390\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00018: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 6ms/step - loss: 0.2068 - sparse_categorical_accuracy: 0.9364\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 6ms/step - loss: 0.4845 - sparse_categorical_accuracy: 0.8317\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 314s 32ms/step - loss: 0.4989 - sparse_categorical_accuracy: 0.8311\n",
            "Load dataset with random state 1\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.32044761 0.36775178 0.31180061]%\n",
            "train P, S and noise labels is [0.30721282 0.38341178 0.3093754 ]%\n",
            "compile the model\n",
            "Fit model on training data step 2\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 7s 16ms/step - loss: 0.6692 - sparse_categorical_accuracy: 0.7184 - val_loss: 0.3443 - val_sparse_categorical_accuracy: 0.8779\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.3118 - sparse_categorical_accuracy: 0.8959 - val_loss: 0.2885 - val_sparse_categorical_accuracy: 0.9034\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2511 - sparse_categorical_accuracy: 0.9199 - val_loss: 0.2527 - val_sparse_categorical_accuracy: 0.9156\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2255 - sparse_categorical_accuracy: 0.9284 - val_loss: 0.2472 - val_sparse_categorical_accuracy: 0.9181\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2057 - sparse_categorical_accuracy: 0.9346 - val_loss: 0.2326 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1953 - sparse_categorical_accuracy: 0.9391 - val_loss: 0.2278 - val_sparse_categorical_accuracy: 0.9293\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1878 - sparse_categorical_accuracy: 0.9428 - val_loss: 0.2344 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1754 - sparse_categorical_accuracy: 0.9459 - val_loss: 0.2151 - val_sparse_categorical_accuracy: 0.9334\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1723 - sparse_categorical_accuracy: 0.9466 - val_loss: 0.2231 - val_sparse_categorical_accuracy: 0.9288\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1641 - sparse_categorical_accuracy: 0.9492 - val_loss: 0.2280 - val_sparse_categorical_accuracy: 0.9303\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1521 - sparse_categorical_accuracy: 0.9538 - val_loss: 0.2193 - val_sparse_categorical_accuracy: 0.9303\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1514 - sparse_categorical_accuracy: 0.9523 - val_loss: 0.2182 - val_sparse_categorical_accuracy: 0.9385\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1465 - sparse_categorical_accuracy: 0.9552 - val_loss: 0.2194 - val_sparse_categorical_accuracy: 0.9369\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00013: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 7ms/step - loss: 0.2150 - sparse_categorical_accuracy: 0.9334\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 6ms/step - loss: 0.4870 - sparse_categorical_accuracy: 0.8339\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.3818 - sparse_categorical_accuracy: 0.8668\n",
            "Load dataset with random state 2\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.31536114 0.3723296  0.31230926]%\n",
            "train P, S and noise labels is [0.30848493 0.38226689 0.30924819]%\n",
            "compile the model\n",
            "Fit model on training data step 3\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 7s 16ms/step - loss: 0.6020 - sparse_categorical_accuracy: 0.7592 - val_loss: 0.3240 - val_sparse_categorical_accuracy: 0.8850\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2921 - sparse_categorical_accuracy: 0.9022 - val_loss: 0.2539 - val_sparse_categorical_accuracy: 0.9084\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2455 - sparse_categorical_accuracy: 0.9214 - val_loss: 0.2252 - val_sparse_categorical_accuracy: 0.9196\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2167 - sparse_categorical_accuracy: 0.9319 - val_loss: 0.2162 - val_sparse_categorical_accuracy: 0.9217\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2026 - sparse_categorical_accuracy: 0.9345 - val_loss: 0.2114 - val_sparse_categorical_accuracy: 0.9278\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1898 - sparse_categorical_accuracy: 0.9380 - val_loss: 0.1993 - val_sparse_categorical_accuracy: 0.9303\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1778 - sparse_categorical_accuracy: 0.9431 - val_loss: 0.2008 - val_sparse_categorical_accuracy: 0.9278\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1708 - sparse_categorical_accuracy: 0.9464 - val_loss: 0.1948 - val_sparse_categorical_accuracy: 0.9329\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1622 - sparse_categorical_accuracy: 0.9501 - val_loss: 0.2020 - val_sparse_categorical_accuracy: 0.9262\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1552 - sparse_categorical_accuracy: 0.9506 - val_loss: 0.1884 - val_sparse_categorical_accuracy: 0.9339\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1462 - sparse_categorical_accuracy: 0.9533 - val_loss: 0.1896 - val_sparse_categorical_accuracy: 0.9344\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1375 - sparse_categorical_accuracy: 0.9581 - val_loss: 0.1911 - val_sparse_categorical_accuracy: 0.9344\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1411 - sparse_categorical_accuracy: 0.9564 - val_loss: 0.1961 - val_sparse_categorical_accuracy: 0.9323\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1349 - sparse_categorical_accuracy: 0.9595 - val_loss: 0.1964 - val_sparse_categorical_accuracy: 0.9359\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1242 - sparse_categorical_accuracy: 0.9626 - val_loss: 0.2052 - val_sparse_categorical_accuracy: 0.9354\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00015: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 7ms/step - loss: 0.1884 - sparse_categorical_accuracy: 0.9334\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 6ms/step - loss: 0.5480 - sparse_categorical_accuracy: 0.8242\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.6422 - sparse_categorical_accuracy: 0.8054\n",
            "Load dataset with random state 3\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30824008 0.3809766  0.31078332]%\n",
            "train P, S and noise labels is [0.31026587 0.38010431 0.30962982]%\n",
            "compile the model\n",
            "Fit model on training data step 4\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 7s 15ms/step - loss: 0.6113 - sparse_categorical_accuracy: 0.7531 - val_loss: 0.3493 - val_sparse_categorical_accuracy: 0.8779\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.3278 - sparse_categorical_accuracy: 0.8853 - val_loss: 0.2690 - val_sparse_categorical_accuracy: 0.9039\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2684 - sparse_categorical_accuracy: 0.9087 - val_loss: 0.2409 - val_sparse_categorical_accuracy: 0.9100\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2406 - sparse_categorical_accuracy: 0.9192 - val_loss: 0.2239 - val_sparse_categorical_accuracy: 0.9212\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2156 - sparse_categorical_accuracy: 0.9284 - val_loss: 0.2028 - val_sparse_categorical_accuracy: 0.9354\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2019 - sparse_categorical_accuracy: 0.9342 - val_loss: 0.1994 - val_sparse_categorical_accuracy: 0.9339\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1863 - sparse_categorical_accuracy: 0.9383 - val_loss: 0.2013 - val_sparse_categorical_accuracy: 0.9379\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1787 - sparse_categorical_accuracy: 0.9434 - val_loss: 0.1850 - val_sparse_categorical_accuracy: 0.9385\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1646 - sparse_categorical_accuracy: 0.9462 - val_loss: 0.1877 - val_sparse_categorical_accuracy: 0.9374\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1605 - sparse_categorical_accuracy: 0.9462 - val_loss: 0.1866 - val_sparse_categorical_accuracy: 0.9410\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1556 - sparse_categorical_accuracy: 0.9504 - val_loss: 0.1895 - val_sparse_categorical_accuracy: 0.9420\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1396 - sparse_categorical_accuracy: 0.9552 - val_loss: 0.1921 - val_sparse_categorical_accuracy: 0.9420\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1399 - sparse_categorical_accuracy: 0.9573 - val_loss: 0.1981 - val_sparse_categorical_accuracy: 0.9349\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00013: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 7ms/step - loss: 0.1850 - sparse_categorical_accuracy: 0.9385\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 6ms/step - loss: 0.5000 - sparse_categorical_accuracy: 0.8312\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.4094 - sparse_categorical_accuracy: 0.8628\n",
            "Load dataset with random state 4\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30773143 0.36978637 0.3224822 ]%\n",
            "train P, S and noise labels is [0.31039308 0.38290294 0.30670398]%\n",
            "compile the model\n",
            "Fit model on training data step 5\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 7s 16ms/step - loss: 0.6214 - sparse_categorical_accuracy: 0.7456 - val_loss: 0.3262 - val_sparse_categorical_accuracy: 0.8800\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.3281 - sparse_categorical_accuracy: 0.8874 - val_loss: 0.2595 - val_sparse_categorical_accuracy: 0.9049\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2720 - sparse_categorical_accuracy: 0.9097 - val_loss: 0.2221 - val_sparse_categorical_accuracy: 0.9242\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2371 - sparse_categorical_accuracy: 0.9179 - val_loss: 0.2070 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2154 - sparse_categorical_accuracy: 0.9281 - val_loss: 0.1916 - val_sparse_categorical_accuracy: 0.9344\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2038 - sparse_categorical_accuracy: 0.9314 - val_loss: 0.1915 - val_sparse_categorical_accuracy: 0.9359\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1897 - sparse_categorical_accuracy: 0.9375 - val_loss: 0.1853 - val_sparse_categorical_accuracy: 0.9420\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1799 - sparse_categorical_accuracy: 0.9403 - val_loss: 0.1814 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1709 - sparse_categorical_accuracy: 0.9422 - val_loss: 0.1897 - val_sparse_categorical_accuracy: 0.9395\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1679 - sparse_categorical_accuracy: 0.9459 - val_loss: 0.1748 - val_sparse_categorical_accuracy: 0.9456\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1515 - sparse_categorical_accuracy: 0.9510 - val_loss: 0.1802 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1481 - sparse_categorical_accuracy: 0.9515 - val_loss: 0.1782 - val_sparse_categorical_accuracy: 0.9435\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1424 - sparse_categorical_accuracy: 0.9528 - val_loss: 0.1925 - val_sparse_categorical_accuracy: 0.9430\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1377 - sparse_categorical_accuracy: 0.9528 - val_loss: 0.1746 - val_sparse_categorical_accuracy: 0.9451\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1343 - sparse_categorical_accuracy: 0.9566 - val_loss: 0.1843 - val_sparse_categorical_accuracy: 0.9446\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1294 - sparse_categorical_accuracy: 0.9581 - val_loss: 0.1756 - val_sparse_categorical_accuracy: 0.9466\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1183 - sparse_categorical_accuracy: 0.9608 - val_loss: 0.1917 - val_sparse_categorical_accuracy: 0.9405\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1171 - sparse_categorical_accuracy: 0.9621 - val_loss: 0.1783 - val_sparse_categorical_accuracy: 0.9496\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1088 - sparse_categorical_accuracy: 0.9623 - val_loss: 0.1764 - val_sparse_categorical_accuracy: 0.9481\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00019: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 6ms/step - loss: 0.1747 - sparse_categorical_accuracy: 0.9451\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 6ms/step - loss: 0.4785 - sparse_categorical_accuracy: 0.8402\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.4188 - sparse_categorical_accuracy: 0.8604\n",
            "Load dataset with random state 5\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30315361 0.37436419 0.3224822 ]%\n",
            "train P, S and noise labels is [0.31153797 0.38175805 0.30670398]%\n",
            "compile the model\n",
            "Fit model on training data step 6\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 7s 16ms/step - loss: 0.5942 - sparse_categorical_accuracy: 0.7561 - val_loss: 0.2912 - val_sparse_categorical_accuracy: 0.9008\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.3199 - sparse_categorical_accuracy: 0.8889 - val_loss: 0.2342 - val_sparse_categorical_accuracy: 0.9237\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.2633 - sparse_categorical_accuracy: 0.9121 - val_loss: 0.2144 - val_sparse_categorical_accuracy: 0.9298\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2337 - sparse_categorical_accuracy: 0.9233 - val_loss: 0.2016 - val_sparse_categorical_accuracy: 0.9323\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2126 - sparse_categorical_accuracy: 0.9312 - val_loss: 0.1965 - val_sparse_categorical_accuracy: 0.9339\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1965 - sparse_categorical_accuracy: 0.9345 - val_loss: 0.1872 - val_sparse_categorical_accuracy: 0.9364\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1821 - sparse_categorical_accuracy: 0.9394 - val_loss: 0.1865 - val_sparse_categorical_accuracy: 0.9395\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1755 - sparse_categorical_accuracy: 0.9433 - val_loss: 0.1855 - val_sparse_categorical_accuracy: 0.9385\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1639 - sparse_categorical_accuracy: 0.9471 - val_loss: 0.1873 - val_sparse_categorical_accuracy: 0.9385\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1569 - sparse_categorical_accuracy: 0.9501 - val_loss: 0.1923 - val_sparse_categorical_accuracy: 0.9390\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1520 - sparse_categorical_accuracy: 0.9501 - val_loss: 0.1876 - val_sparse_categorical_accuracy: 0.9446\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1410 - sparse_categorical_accuracy: 0.9562 - val_loss: 0.1846 - val_sparse_categorical_accuracy: 0.9420\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1371 - sparse_categorical_accuracy: 0.9551 - val_loss: 0.2028 - val_sparse_categorical_accuracy: 0.9385\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1280 - sparse_categorical_accuracy: 0.9589 - val_loss: 0.1961 - val_sparse_categorical_accuracy: 0.9430\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1217 - sparse_categorical_accuracy: 0.9625 - val_loss: 0.1943 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1180 - sparse_categorical_accuracy: 0.9616 - val_loss: 0.1978 - val_sparse_categorical_accuracy: 0.9405\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1057 - sparse_categorical_accuracy: 0.9665 - val_loss: 0.2315 - val_sparse_categorical_accuracy: 0.9374\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00017: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 7ms/step - loss: 0.1845 - sparse_categorical_accuracy: 0.9420\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 7ms/step - loss: 0.4812 - sparse_categorical_accuracy: 0.8423\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.3796 - sparse_categorical_accuracy: 0.8762\n",
            "Load dataset with random state 6\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30620549 0.37945066 0.31434385]%\n",
            "train P, S and noise labels is [0.31077471 0.38048594 0.30873935]%\n",
            "compile the model\n",
            "Fit model on training data step 7\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 7s 16ms/step - loss: 0.6268 - sparse_categorical_accuracy: 0.7596 - val_loss: 0.3454 - val_sparse_categorical_accuracy: 0.8845\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.3174 - sparse_categorical_accuracy: 0.8952 - val_loss: 0.2664 - val_sparse_categorical_accuracy: 0.9079\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2537 - sparse_categorical_accuracy: 0.9163 - val_loss: 0.2409 - val_sparse_categorical_accuracy: 0.9166\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2221 - sparse_categorical_accuracy: 0.9267 - val_loss: 0.2411 - val_sparse_categorical_accuracy: 0.9156\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2055 - sparse_categorical_accuracy: 0.9331 - val_loss: 0.2174 - val_sparse_categorical_accuracy: 0.9262\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1912 - sparse_categorical_accuracy: 0.9387 - val_loss: 0.2256 - val_sparse_categorical_accuracy: 0.9227\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1772 - sparse_categorical_accuracy: 0.9425 - val_loss: 0.2190 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1734 - sparse_categorical_accuracy: 0.9461 - val_loss: 0.2030 - val_sparse_categorical_accuracy: 0.9349\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1588 - sparse_categorical_accuracy: 0.9522 - val_loss: 0.2065 - val_sparse_categorical_accuracy: 0.9303\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1542 - sparse_categorical_accuracy: 0.9501 - val_loss: 0.2091 - val_sparse_categorical_accuracy: 0.9323\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1448 - sparse_categorical_accuracy: 0.9532 - val_loss: 0.2087 - val_sparse_categorical_accuracy: 0.9349\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1399 - sparse_categorical_accuracy: 0.9566 - val_loss: 0.2107 - val_sparse_categorical_accuracy: 0.9359\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1315 - sparse_categorical_accuracy: 0.9584 - val_loss: 0.2044 - val_sparse_categorical_accuracy: 0.9385\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00013: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 7ms/step - loss: 0.2030 - sparse_categorical_accuracy: 0.9349\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 7ms/step - loss: 0.4622 - sparse_categorical_accuracy: 0.8447\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.3878 - sparse_categorical_accuracy: 0.8716\n",
            "Load dataset with random state 7\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.29857579 0.3819939  0.31943032]%\n",
            "train P, S and noise labels is [0.31268286 0.37984989 0.30746724]%\n",
            "compile the model\n",
            "Fit model on training data step 8\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 7s 16ms/step - loss: 0.6232 - sparse_categorical_accuracy: 0.7442 - val_loss: 0.3660 - val_sparse_categorical_accuracy: 0.8647\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.3320 - sparse_categorical_accuracy: 0.8879 - val_loss: 0.2816 - val_sparse_categorical_accuracy: 0.8973\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2723 - sparse_categorical_accuracy: 0.9075 - val_loss: 0.2520 - val_sparse_categorical_accuracy: 0.9090\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2351 - sparse_categorical_accuracy: 0.9227 - val_loss: 0.2375 - val_sparse_categorical_accuracy: 0.9130\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2179 - sparse_categorical_accuracy: 0.9303 - val_loss: 0.2286 - val_sparse_categorical_accuracy: 0.9217\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2035 - sparse_categorical_accuracy: 0.9336 - val_loss: 0.2213 - val_sparse_categorical_accuracy: 0.9268\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1856 - sparse_categorical_accuracy: 0.9401 - val_loss: 0.2089 - val_sparse_categorical_accuracy: 0.9293\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1790 - sparse_categorical_accuracy: 0.9430 - val_loss: 0.2071 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1662 - sparse_categorical_accuracy: 0.9468 - val_loss: 0.2079 - val_sparse_categorical_accuracy: 0.9318\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1579 - sparse_categorical_accuracy: 0.9514 - val_loss: 0.2025 - val_sparse_categorical_accuracy: 0.9344\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1513 - sparse_categorical_accuracy: 0.9504 - val_loss: 0.2033 - val_sparse_categorical_accuracy: 0.9339\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1387 - sparse_categorical_accuracy: 0.9560 - val_loss: 0.2101 - val_sparse_categorical_accuracy: 0.9369\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1364 - sparse_categorical_accuracy: 0.9553 - val_loss: 0.2059 - val_sparse_categorical_accuracy: 0.9313\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1330 - sparse_categorical_accuracy: 0.9566 - val_loss: 0.2124 - val_sparse_categorical_accuracy: 0.9369\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1268 - sparse_categorical_accuracy: 0.9584 - val_loss: 0.2148 - val_sparse_categorical_accuracy: 0.9288\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00015: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 7ms/step - loss: 0.2025 - sparse_categorical_accuracy: 0.9344\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 7ms/step - loss: 0.5220 - sparse_categorical_accuracy: 0.8284\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 312s 32ms/step - loss: 0.3987 - sparse_categorical_accuracy: 0.8687\n",
            "Load dataset with random state 8\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30417091 0.38046796 0.31536114]%\n",
            "train P, S and noise labels is [0.31128355 0.38023152 0.30848493]%\n",
            "compile the model\n",
            "Fit model on training data step 9\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 8s 16ms/step - loss: 0.6049 - sparse_categorical_accuracy: 0.7671 - val_loss: 0.3443 - val_sparse_categorical_accuracy: 0.8754\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.3219 - sparse_categorical_accuracy: 0.8919 - val_loss: 0.2654 - val_sparse_categorical_accuracy: 0.9059\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2536 - sparse_categorical_accuracy: 0.9160 - val_loss: 0.2597 - val_sparse_categorical_accuracy: 0.9156\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2254 - sparse_categorical_accuracy: 0.9243 - val_loss: 0.2400 - val_sparse_categorical_accuracy: 0.9217\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2077 - sparse_categorical_accuracy: 0.9341 - val_loss: 0.2327 - val_sparse_categorical_accuracy: 0.9247\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1897 - sparse_categorical_accuracy: 0.9394 - val_loss: 0.2321 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1750 - sparse_categorical_accuracy: 0.9449 - val_loss: 0.2290 - val_sparse_categorical_accuracy: 0.9288\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1668 - sparse_categorical_accuracy: 0.9478 - val_loss: 0.2463 - val_sparse_categorical_accuracy: 0.9222\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1552 - sparse_categorical_accuracy: 0.9509 - val_loss: 0.2180 - val_sparse_categorical_accuracy: 0.9349\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1533 - sparse_categorical_accuracy: 0.9536 - val_loss: 0.2246 - val_sparse_categorical_accuracy: 0.9339\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1437 - sparse_categorical_accuracy: 0.9569 - val_loss: 0.2283 - val_sparse_categorical_accuracy: 0.9334\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1362 - sparse_categorical_accuracy: 0.9575 - val_loss: 0.2342 - val_sparse_categorical_accuracy: 0.9334\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1294 - sparse_categorical_accuracy: 0.9604 - val_loss: 0.2355 - val_sparse_categorical_accuracy: 0.9268\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1258 - sparse_categorical_accuracy: 0.9613 - val_loss: 0.2396 - val_sparse_categorical_accuracy: 0.9318\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00014: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 7ms/step - loss: 0.2180 - sparse_categorical_accuracy: 0.9349\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 7ms/step - loss: 0.4907 - sparse_categorical_accuracy: 0.8391\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 312s 32ms/step - loss: 0.4794 - sparse_categorical_accuracy: 0.8510\n",
            "Load dataset with random state 9\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.31892167 0.37487284 0.30620549]%\n",
            "train P, S and noise labels is [0.30759445 0.38163084 0.31077471]%\n",
            "compile the model\n",
            "Fit model on training data step 10\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 7s 16ms/step - loss: 0.6359 - sparse_categorical_accuracy: 0.7434 - val_loss: 0.3602 - val_sparse_categorical_accuracy: 0.8708\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.3348 - sparse_categorical_accuracy: 0.8842 - val_loss: 0.2877 - val_sparse_categorical_accuracy: 0.9023\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.2735 - sparse_categorical_accuracy: 0.9062 - val_loss: 0.2638 - val_sparse_categorical_accuracy: 0.9120\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.2511 - sparse_categorical_accuracy: 0.9183 - val_loss: 0.2482 - val_sparse_categorical_accuracy: 0.9135\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.2334 - sparse_categorical_accuracy: 0.9223 - val_loss: 0.2339 - val_sparse_categorical_accuracy: 0.9196\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.2139 - sparse_categorical_accuracy: 0.9289 - val_loss: 0.2197 - val_sparse_categorical_accuracy: 0.9278\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1989 - sparse_categorical_accuracy: 0.9335 - val_loss: 0.2178 - val_sparse_categorical_accuracy: 0.9273\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1895 - sparse_categorical_accuracy: 0.9396 - val_loss: 0.2083 - val_sparse_categorical_accuracy: 0.9298\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1796 - sparse_categorical_accuracy: 0.9425 - val_loss: 0.2146 - val_sparse_categorical_accuracy: 0.9298\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1695 - sparse_categorical_accuracy: 0.9448 - val_loss: 0.2208 - val_sparse_categorical_accuracy: 0.9273\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1601 - sparse_categorical_accuracy: 0.9487 - val_loss: 0.2111 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1535 - sparse_categorical_accuracy: 0.9520 - val_loss: 0.1995 - val_sparse_categorical_accuracy: 0.9354\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1435 - sparse_categorical_accuracy: 0.9546 - val_loss: 0.2053 - val_sparse_categorical_accuracy: 0.9339\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1414 - sparse_categorical_accuracy: 0.9538 - val_loss: 0.1991 - val_sparse_categorical_accuracy: 0.9354\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 3s 14ms/step - loss: 0.1327 - sparse_categorical_accuracy: 0.9587 - val_loss: 0.2031 - val_sparse_categorical_accuracy: 0.9390\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1313 - sparse_categorical_accuracy: 0.9578 - val_loss: 0.1934 - val_sparse_categorical_accuracy: 0.9379\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1227 - sparse_categorical_accuracy: 0.9615 - val_loss: 0.2042 - val_sparse_categorical_accuracy: 0.9385\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1165 - sparse_categorical_accuracy: 0.9634 - val_loss: 0.2083 - val_sparse_categorical_accuracy: 0.9400\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1154 - sparse_categorical_accuracy: 0.9637 - val_loss: 0.2057 - val_sparse_categorical_accuracy: 0.9349\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1120 - sparse_categorical_accuracy: 0.9660 - val_loss: 0.2089 - val_sparse_categorical_accuracy: 0.9385\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 3s 13ms/step - loss: 0.1081 - sparse_categorical_accuracy: 0.9697 - val_loss: 0.2042 - val_sparse_categorical_accuracy: 0.9410\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00021: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 7ms/step - loss: 0.1931 - sparse_categorical_accuracy: 0.9379\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 7ms/step - loss: 0.4721 - sparse_categorical_accuracy: 0.8477\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 312s 32ms/step - loss: 0.3828 - sparse_categorical_accuracy: 0.8652\n",
            "CPU times: user 36min 17s, sys: 8min 23s, total: 44min 40s\n",
            "Wall time: 1h 3min 56s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5LtKdSmDBLXF"
      },
      "source": [
        "### Print collected values and scores\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iyF7HFxTBSSq",
        "outputId": "e86d6840-74b7-45a6-de6b-426704ca6d52"
      },
      "source": [
        "print(val_sparse_categorical_accuracy)\n",
        "print(val_loss)\n",
        "print(sakh_top_1)\n",
        "print(dag_top_1)\n",
        "print(cal_top_1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.882502555847168, 0.9079349040985107, 0.9175992012023926, 0.9237029552459717, 0.928280770778656, 0.9277721047401428, 0.9237029552459717, 0.9338759183883667, 0.9328585863113403, 0.9328585863113403, 0.9359104633331299, 0.9359104633331299, 0.9364191293716431, 0.9399796724319458, 0.9425228834152222, 0.9394710063934326, 0.9399796724319458, 0.9389623403549194], [0.8779247403144836, 0.9033570885658264, 0.9155645966529846, 0.918107807636261, 0.928280770778656, 0.9292980432510376, 0.928280770778656, 0.9333672523498535, 0.9287894368171692, 0.930315375328064, 0.930315375328064, 0.938453733921051, 0.9369277954101562], [0.8850457668304443, 0.9084435105323792, 0.9196337461471558, 0.9216683506965637, 0.9277721047401428, 0.930315375328064, 0.9277721047401428, 0.9328585863113403, 0.926246166229248, 0.9338759183883667, 0.9343845248222351, 0.9343845248222351, 0.9323499202728271, 0.9359104633331299, 0.9354018568992615], [0.8779247403144836, 0.9038656949996948, 0.9099695086479187, 0.9211597442626953, 0.9354018568992615, 0.9338759183883667, 0.9379450678825378, 0.938453733921051, 0.9374364018440247, 0.9409969449043274, 0.942014217376709, 0.942014217376709, 0.9348931908607483], [0.8799592852592468, 0.9048830270767212, 0.9242116212844849, 0.928280770778656, 0.9343845248222351, 0.9359104633331299, 0.942014217376709, 0.9425228834152222, 0.9394710063934326, 0.9455747604370117, 0.9440488219261169, 0.9435401558876038, 0.9430315494537354, 0.9450660943984985, 0.9445574879646301, 0.9465920925140381, 0.9404882788658142, 0.9496439695358276, 0.9481180310249329], [0.9008138179779053, 0.9237029552459717, 0.9298067092895508, 0.9323499202728271, 0.9338759183883667, 0.9364191293716431, 0.9394710063934326, 0.938453733921051, 0.938453733921051, 0.9389623403549194, 0.9445574879646301, 0.942014217376709, 0.938453733921051, 0.9430315494537354, 0.9440488219261169, 0.9404882788658142, 0.9374364018440247], [0.8845371603965759, 0.9079349040985107, 0.9165818691253662, 0.9155645966529846, 0.926246166229248, 0.9226856827735901, 0.928280770778656, 0.9348931908607483, 0.930315375328064, 0.9323499202728271, 0.9348931908607483, 0.9359104633331299, 0.938453733921051], [0.8646999001502991, 0.8972533345222473, 0.9089521765708923, 0.9130213856697083, 0.9216683506965637, 0.9267548322677612, 0.9292980432510376, 0.928280770778656, 0.9318413138389587, 0.9343845248222351, 0.9338759183883667, 0.9369277954101562, 0.9313326478004456, 0.9369277954101562, 0.9287894368171692], [0.8753814697265625, 0.9059002995491028, 0.9155645966529846, 0.9216683506965637, 0.9247202277183533, 0.928280770778656, 0.9287894368171692, 0.9221770167350769, 0.9348931908607483, 0.9338759183883667, 0.9333672523498535, 0.9333672523498535, 0.9267548322677612, 0.9318413138389587], [0.8708036541938782, 0.9023397564888, 0.9120040535926819, 0.9135299921035767, 0.9196337461471558, 0.9277721047401428, 0.9272634983062744, 0.9298067092895508, 0.9298067092895508, 0.9272634983062744, 0.928280770778656, 0.9354018568992615, 0.9338759183883667, 0.9354018568992615, 0.9389623403549194, 0.9379450678825378, 0.938453733921051, 0.9399796724319458, 0.9348931908607483, 0.938453733921051, 0.9409969449043274]]\n",
            "[[0.34258100390434265, 0.27411705255508423, 0.2456008940935135, 0.24949400126934052, 0.22967149317264557, 0.23146995902061462, 0.2241021692752838, 0.2135114222764969, 0.21483393013477325, 0.2165006548166275, 0.21531350910663605, 0.20955950021743774, 0.20678552985191345, 0.2133357673883438, 0.20884600281715393, 0.22261741757392883, 0.2128853052854538, 0.22069716453552246], [0.34428322315216064, 0.2885216176509857, 0.25273022055625916, 0.2471965104341507, 0.23261435329914093, 0.22779560089111328, 0.23439283668994904, 0.21505635976791382, 0.2230989933013916, 0.22796247899532318, 0.21926718950271606, 0.2182433307170868, 0.21940910816192627], [0.32398808002471924, 0.2539164423942566, 0.22524070739746094, 0.21615996956825256, 0.21142394840717316, 0.19930918514728546, 0.20076748728752136, 0.1947704255580902, 0.20197826623916626, 0.1884460747241974, 0.1895918846130371, 0.19112616777420044, 0.1960769146680832, 0.1964147388935089, 0.20520004630088806], [0.349263995885849, 0.26901572942733765, 0.24089781939983368, 0.22385698556900024, 0.2027554214000702, 0.19935064017772675, 0.20126992464065552, 0.18503858149051666, 0.1876675933599472, 0.18656125664710999, 0.18945255875587463, 0.1921052485704422, 0.1980624794960022], [0.3262290954589844, 0.2595202326774597, 0.22213461995124817, 0.20697355270385742, 0.1915997713804245, 0.19152021408081055, 0.18525829911231995, 0.18138685822486877, 0.189731627702713, 0.17483285069465637, 0.1802445948123932, 0.17819607257843018, 0.19253355264663696, 0.17461250722408295, 0.18429630994796753, 0.17560142278671265, 0.19172145426273346, 0.17827193439006805, 0.17639175057411194], [0.2911534011363983, 0.2341753989458084, 0.21443606913089752, 0.2015921175479889, 0.1964949667453766, 0.18719859421253204, 0.18648387491703033, 0.18547670543193817, 0.18729303777217865, 0.19228851795196533, 0.1876295953989029, 0.18461810052394867, 0.20276163518428802, 0.1960652470588684, 0.19431030750274658, 0.19782066345214844, 0.2315201312303543], [0.34539595246315, 0.2664142847061157, 0.24093575775623322, 0.24108006060123444, 0.2173883616924286, 0.22562986612319946, 0.2190379500389099, 0.2029627412557602, 0.20647738873958588, 0.20913393795490265, 0.20872941613197327, 0.21071434020996094, 0.20441891252994537], [0.36604857444763184, 0.281625896692276, 0.2520233690738678, 0.2374943345785141, 0.22859781980514526, 0.22126685082912445, 0.20894719660282135, 0.20713648200035095, 0.20791307091712952, 0.20254482328891754, 0.20326071977615356, 0.21009854972362518, 0.20594555139541626, 0.21235761046409607, 0.21482916176319122], [0.344279944896698, 0.26535797119140625, 0.25970664620399475, 0.2399968057870865, 0.23274241387844086, 0.23205751180648804, 0.2289925068616867, 0.2462555170059204, 0.21804997324943542, 0.22456194460391998, 0.22832559049129486, 0.23415741324424744, 0.2354736626148224, 0.23956619203090668], [0.3602103590965271, 0.2876826226711273, 0.2638483941555023, 0.24822834134101868, 0.23392246663570404, 0.21970710158348083, 0.2177661657333374, 0.20831730961799622, 0.21458128094673157, 0.2207888662815094, 0.2110653519630432, 0.19947293400764465, 0.20532071590423584, 0.19912415742874146, 0.2031172811985016, 0.19341707229614258, 0.2042074054479599, 0.20827774703502655, 0.20565621554851532, 0.2089415192604065, 0.2042076140642166]]\n",
            "[0.9364191293716431, 0.9333672523498535, 0.9333672523498535, 0.938453733921051, 0.9450660943984985, 0.942014217376709, 0.9348931908607483, 0.9343845248222351, 0.9348931908607483, 0.9379450678825378]\n",
            "[0.8317283391952515, 0.8338842988014221, 0.8241825103759766, 0.8311893343925476, 0.8402443528175354, 0.8422924876213074, 0.8446999788284302, 0.8283507227897644, 0.8390945196151733, 0.8477182984352112]\n",
            "[0.8311281204223633, 0.86683189868927, 0.8054448962211609, 0.8628442883491516, 0.8604274392127991, 0.8761572241783142, 0.8715764284133911, 0.8687489628791809, 0.8510139584541321, 0.8652117848396301]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rCq4SzWGBVSv",
        "outputId": "efd56e47-0eda-418c-8427-fe6614193e74"
      },
      "source": [
        "print(\"Mean SAKHALIN test TOP-1 accuracy {}, std. dev {}\".format(np.mean(sakh_top_1), np.std(sakh_top_1)))\n",
        "print(\"Mean DAGESTAN test TOP-1 accuracy {}, std. dev {}\".format(np.mean(dag_top_1), np.std(dag_top_1)))\n",
        "print(\"Mean CALIFIRNIA test TOP-1 accuracy {}, std. dev {}\".format(np.mean(cal_top_1), np.std(cal_top_1)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mean SAKHALIN test TOP-1 accuracy 0.9370803654193878, std. dev 0.0036823265190096283\n",
            "Mean DAGESTAN test TOP-1 accuracy 0.836338484287262, std. dev 0.007232417996005477\n",
            "Mean CALIFIRNIA test TOP-1 accuracy 0.8559385001659393, std. dev 0.02068661420614195\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HdXHDsHgTJqY"
      },
      "source": [
        "# Train Spec-CNN on Sakhalin Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3EmejTW_TJqi"
      },
      "source": [
        "### model spec"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RnspP0zVTJqi",
        "outputId": "2132324a-7304-49cc-f37e-fc8bf4a403ff"
      },
      "source": [
        "model_cnn = model_cnn_spec(400, 64, 16)\n",
        "\n",
        "LR = 0.0001\n",
        "model_cnn.compile(\n",
        "    optimizer=keras.optimizers.Adam(learning_rate=LR),\n",
        "    loss=keras.losses.SparseCategoricalCrossentropy(),\n",
        "    metrics=[keras.metrics.SparseCategoricalAccuracy()],)\n",
        "\n",
        "model_cnn.summary() "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 400, 3)]          0         \n",
            "_________________________________________________________________\n",
            "stft (STFT)                  (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude (Magnitude)        (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude_to_decibel (Magnit (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "max_abs_scaler (MaxABSScaler (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "conv2d (Conv2D)              (None, 22, 33, 32)        896       \n",
            "_________________________________________________________________\n",
            "batch_normalization (BatchNo (None, 22, 33, 32)        128       \n",
            "_________________________________________________________________\n",
            "tf.nn.relu (TFOpLambda)      (None, 22, 33, 32)        0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D) (None, 11, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 11, 16, 64)        18496     \n",
            "_________________________________________________________________\n",
            "batch_normalization_1 (Batch (None, 11, 16, 64)        256       \n",
            "_________________________________________________________________\n",
            "tf.nn.relu_1 (TFOpLambda)    (None, 11, 16, 64)        0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2 (None, 5, 8, 64)          0         \n",
            "_________________________________________________________________\n",
            "conv2d_2 (Conv2D)            (None, 5, 8, 128)         73856     \n",
            "_________________________________________________________________\n",
            "batch_normalization_2 (Batch (None, 5, 8, 128)         512       \n",
            "_________________________________________________________________\n",
            "tf.nn.relu_2 (TFOpLambda)    (None, 5, 8, 128)         0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2 (None, 2, 4, 128)         0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 80)                82000     \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 80)                0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 3)                 243       \n",
            "=================================================================\n",
            "Total params: 176,387\n",
            "Trainable params: 175,939\n",
            "Non-trainable params: 448\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "66qhpmCLTJqj"
      },
      "source": [
        "### Train the model with different random seeds"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MZRKXlqBTJqk",
        "outputId": "d89c857a-f31a-4f39-a6d3-8d5e5a2b4432"
      },
      "source": [
        "%%time\n",
        "\n",
        "interations = 10\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 5, restore_best_weights = True, verbose = 1),\n",
        "    keras.callbacks.ModelCheckpoint('w_cnn_epoch_hpa{epoch}.h5', save_weights_only=True),\n",
        "]\n",
        "\n",
        "sparse_categorical_accuracy = []\n",
        "loss = []\n",
        "val_loss = []\n",
        "val_sparse_categorical_accuracy = []\n",
        "\n",
        "cal_top_1 = []\n",
        "sakh_top_1 = []\n",
        "dag_top_1 = []\n",
        "\n",
        "# load California data for test\n",
        "X_train, X_test = h5_tts('/content/scsn_ps_2000_2017_shuf.hdf5',\\\n",
        "                             batch_size = 480,\\\n",
        "                             test_size = 0.99, random_state = 0, shuffle = False)\n",
        "\n",
        "\n",
        "for i in range(interations):\n",
        "  print(\"Load dataset with random state {}\".format(i))\n",
        "  X_train_sakh, X_test_sakh, y_train_sakh, y_test_sakh =\\\n",
        "      load_test_train_data('/content/drive/MyDrive/DATA/ML_DATASETS_SAKH_DAG/SAKH-2014_2021_global_norm.h5', 0.2, random_state=i)   \n",
        "\n",
        "  print(\"compile the model\")\n",
        "  model_cnn = model_cnn_spec(400, 64, 16)\n",
        "\n",
        "  LR = 0.0001\n",
        "  model_cnn.compile(\n",
        "      optimizer=keras.optimizers.Adam(learning_rate=LR),\n",
        "      loss=keras.losses.SparseCategoricalCrossentropy(),\n",
        "      metrics=[keras.metrics.SparseCategoricalAccuracy()],)\n",
        "\n",
        "  print(\"Fit model on training data step {}\".format(i+1))\n",
        "  history = model_cnn.fit(\n",
        "    X_train_sakh,\n",
        "    y_train_sakh,\n",
        "    #shuffle=False,\n",
        "    epochs=400,\n",
        "    validation_data=(X_test_sakh, y_test_sakh),\n",
        "    callbacks = callbacks\n",
        "    )\n",
        "  \n",
        "  #save convergance curve \n",
        "  sparse_categorical_accuracy.append(history.history['sparse_categorical_accuracy'])\n",
        "  loss.append(history.history['loss'])\n",
        "  val_sparse_categorical_accuracy.append(history.history['val_sparse_categorical_accuracy'])\n",
        "  val_loss.append(history.history['val_loss'])\n",
        "\n",
        "  print(\"Evaluate transformer based model on SAKHALIN test data\")\n",
        "  sakh_top_1.append(model_cnn.evaluate(X_test_sakh, y_test_sakh, batch_size=128)[1])\n",
        "\n",
        "  print(\"Evaluate transformer based model on DAGESTAN test data\")\n",
        "  dag_top_1.append(model_cnn.evaluate(X_test_dag, y_test_dag, batch_size=128)[1])\n",
        "  \n",
        "  print(\"Evaluate transformer based model on CALIFORNIA test data\")\n",
        "  cal_top_1.append(model_cnn.evaluate(X_test)[1])\n",
        "\n",
        "  model_cnn.save_weights('/content/drive/MyDrive/MODELS/w_model_cnn_with_spec_sakh_seeds.hd5', save_format='h5')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Load dataset with random state 0\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.28992879 0.39572737 0.31434385]%\n",
            "train P, S and noise labels is [0.31484544 0.37641521 0.30873935]%\n",
            "compile the model\n",
            "Fit model on training data step 1\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 19s 7ms/step - loss: 1.0124 - sparse_categorical_accuracy: 0.6016 - val_loss: 1.4814 - val_sparse_categorical_accuracy: 0.2899\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.6015 - sparse_categorical_accuracy: 0.7513 - val_loss: 0.7338 - val_sparse_categorical_accuracy: 0.7040\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.5007 - sparse_categorical_accuracy: 0.8007 - val_loss: 0.3683 - val_sparse_categorical_accuracy: 0.8693\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4312 - sparse_categorical_accuracy: 0.8307 - val_loss: 0.3135 - val_sparse_categorical_accuracy: 0.8876\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3827 - sparse_categorical_accuracy: 0.8563 - val_loss: 0.2864 - val_sparse_categorical_accuracy: 0.9044\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3572 - sparse_categorical_accuracy: 0.8634 - val_loss: 0.2626 - val_sparse_categorical_accuracy: 0.9079\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3257 - sparse_categorical_accuracy: 0.8788 - val_loss: 0.2425 - val_sparse_categorical_accuracy: 0.9212\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2937 - sparse_categorical_accuracy: 0.8928 - val_loss: 0.2276 - val_sparse_categorical_accuracy: 0.9212\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2819 - sparse_categorical_accuracy: 0.8953 - val_loss: 0.2188 - val_sparse_categorical_accuracy: 0.9268\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2529 - sparse_categorical_accuracy: 0.9087 - val_loss: 0.2062 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2409 - sparse_categorical_accuracy: 0.9169 - val_loss: 0.2083 - val_sparse_categorical_accuracy: 0.9308\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2369 - sparse_categorical_accuracy: 0.9172 - val_loss: 0.1989 - val_sparse_categorical_accuracy: 0.9395\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2241 - sparse_categorical_accuracy: 0.9174 - val_loss: 0.1953 - val_sparse_categorical_accuracy: 0.9410\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2180 - sparse_categorical_accuracy: 0.9260 - val_loss: 0.1874 - val_sparse_categorical_accuracy: 0.9385\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2177 - sparse_categorical_accuracy: 0.9260 - val_loss: 0.2007 - val_sparse_categorical_accuracy: 0.9334\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1998 - sparse_categorical_accuracy: 0.9289 - val_loss: 0.1766 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1964 - sparse_categorical_accuracy: 0.9351 - val_loss: 0.1744 - val_sparse_categorical_accuracy: 0.9430\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1878 - sparse_categorical_accuracy: 0.9355 - val_loss: 0.1720 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1786 - sparse_categorical_accuracy: 0.9398 - val_loss: 0.1723 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1756 - sparse_categorical_accuracy: 0.9398 - val_loss: 0.1696 - val_sparse_categorical_accuracy: 0.9461\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1702 - sparse_categorical_accuracy: 0.9425 - val_loss: 0.1678 - val_sparse_categorical_accuracy: 0.9476\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1679 - sparse_categorical_accuracy: 0.9443 - val_loss: 0.2110 - val_sparse_categorical_accuracy: 0.9339\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1572 - sparse_categorical_accuracy: 0.9461 - val_loss: 0.1713 - val_sparse_categorical_accuracy: 0.9466\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1467 - sparse_categorical_accuracy: 0.9485 - val_loss: 0.1634 - val_sparse_categorical_accuracy: 0.9491\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1523 - sparse_categorical_accuracy: 0.9506 - val_loss: 0.1673 - val_sparse_categorical_accuracy: 0.9507\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1494 - sparse_categorical_accuracy: 0.9475 - val_loss: 0.1777 - val_sparse_categorical_accuracy: 0.9476\n",
            "Epoch 27/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1479 - sparse_categorical_accuracy: 0.9496 - val_loss: 0.1635 - val_sparse_categorical_accuracy: 0.9517\n",
            "Epoch 28/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1340 - sparse_categorical_accuracy: 0.9542 - val_loss: 0.1623 - val_sparse_categorical_accuracy: 0.9502\n",
            "Epoch 29/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1303 - sparse_categorical_accuracy: 0.9569 - val_loss: 0.1625 - val_sparse_categorical_accuracy: 0.9512\n",
            "Epoch 30/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1298 - sparse_categorical_accuracy: 0.9583 - val_loss: 0.1677 - val_sparse_categorical_accuracy: 0.9476\n",
            "Epoch 31/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1245 - sparse_categorical_accuracy: 0.9574 - val_loss: 0.1743 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 32/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1207 - sparse_categorical_accuracy: 0.9590 - val_loss: 0.1623 - val_sparse_categorical_accuracy: 0.9527\n",
            "Epoch 33/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1180 - sparse_categorical_accuracy: 0.9604 - val_loss: 0.1745 - val_sparse_categorical_accuracy: 0.9451\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00033: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 5ms/step - loss: 0.1621 - sparse_categorical_accuracy: 0.9502\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4305 - sparse_categorical_accuracy: 0.8670\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.4224 - sparse_categorical_accuracy: 0.8518\n",
            "Load dataset with random state 1\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.32044761 0.36775178 0.31180061]%\n",
            "train P, S and noise labels is [0.30721282 0.38341178 0.3093754 ]%\n",
            "compile the model\n",
            "Fit model on training data step 2\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 3s 7ms/step - loss: 0.9350 - sparse_categorical_accuracy: 0.6306 - val_loss: 0.9766 - val_sparse_categorical_accuracy: 0.4039\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.5487 - sparse_categorical_accuracy: 0.7812 - val_loss: 0.5015 - val_sparse_categorical_accuracy: 0.8138\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4538 - sparse_categorical_accuracy: 0.8255 - val_loss: 0.3364 - val_sparse_categorical_accuracy: 0.8927\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3855 - sparse_categorical_accuracy: 0.8542 - val_loss: 0.2870 - val_sparse_categorical_accuracy: 0.9095\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3510 - sparse_categorical_accuracy: 0.8705 - val_loss: 0.2600 - val_sparse_categorical_accuracy: 0.9171\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3255 - sparse_categorical_accuracy: 0.8814 - val_loss: 0.2489 - val_sparse_categorical_accuracy: 0.9161\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 5ms/step - loss: 0.2936 - sparse_categorical_accuracy: 0.8948 - val_loss: 0.2316 - val_sparse_categorical_accuracy: 0.9298\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2786 - sparse_categorical_accuracy: 0.9036 - val_loss: 0.2077 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2625 - sparse_categorical_accuracy: 0.9118 - val_loss: 0.2070 - val_sparse_categorical_accuracy: 0.9323\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2471 - sparse_categorical_accuracy: 0.9115 - val_loss: 0.1971 - val_sparse_categorical_accuracy: 0.9354\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2370 - sparse_categorical_accuracy: 0.9177 - val_loss: 0.1881 - val_sparse_categorical_accuracy: 0.9390\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2272 - sparse_categorical_accuracy: 0.9234 - val_loss: 0.1830 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2104 - sparse_categorical_accuracy: 0.9290 - val_loss: 0.1764 - val_sparse_categorical_accuracy: 0.9496\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1966 - sparse_categorical_accuracy: 0.9312 - val_loss: 0.1745 - val_sparse_categorical_accuracy: 0.9461\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2049 - sparse_categorical_accuracy: 0.9276 - val_loss: 0.1701 - val_sparse_categorical_accuracy: 0.9456\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1909 - sparse_categorical_accuracy: 0.9335 - val_loss: 0.1663 - val_sparse_categorical_accuracy: 0.9466\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1932 - sparse_categorical_accuracy: 0.9350 - val_loss: 0.1666 - val_sparse_categorical_accuracy: 0.9512\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1805 - sparse_categorical_accuracy: 0.9370 - val_loss: 0.1680 - val_sparse_categorical_accuracy: 0.9476\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1770 - sparse_categorical_accuracy: 0.9401 - val_loss: 0.1614 - val_sparse_categorical_accuracy: 0.9537\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1735 - sparse_categorical_accuracy: 0.9389 - val_loss: 0.1647 - val_sparse_categorical_accuracy: 0.9481\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1674 - sparse_categorical_accuracy: 0.9435 - val_loss: 0.1562 - val_sparse_categorical_accuracy: 0.9512\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1546 - sparse_categorical_accuracy: 0.9473 - val_loss: 0.1689 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1487 - sparse_categorical_accuracy: 0.9506 - val_loss: 0.1547 - val_sparse_categorical_accuracy: 0.9527\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1565 - sparse_categorical_accuracy: 0.9487 - val_loss: 0.1611 - val_sparse_categorical_accuracy: 0.9456\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1454 - sparse_categorical_accuracy: 0.9505 - val_loss: 0.1614 - val_sparse_categorical_accuracy: 0.9476\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1445 - sparse_categorical_accuracy: 0.9500 - val_loss: 0.1673 - val_sparse_categorical_accuracy: 0.9496\n",
            "Epoch 27/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1401 - sparse_categorical_accuracy: 0.9528 - val_loss: 0.1740 - val_sparse_categorical_accuracy: 0.9471\n",
            "Epoch 28/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1354 - sparse_categorical_accuracy: 0.9522 - val_loss: 0.1577 - val_sparse_categorical_accuracy: 0.9491\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00028: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 4ms/step - loss: 0.1547 - sparse_categorical_accuracy: 0.9527\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4159 - sparse_categorical_accuracy: 0.8604\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.4862 - sparse_categorical_accuracy: 0.8393\n",
            "Load dataset with random state 2\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.31536114 0.3723296  0.31230926]%\n",
            "train P, S and noise labels is [0.30848493 0.38226689 0.30924819]%\n",
            "compile the model\n",
            "Fit model on training data step 3\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 3s 7ms/step - loss: 0.9786 - sparse_categorical_accuracy: 0.5905 - val_loss: 1.0223 - val_sparse_categorical_accuracy: 0.3703\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.5974 - sparse_categorical_accuracy: 0.7569 - val_loss: 0.5420 - val_sparse_categorical_accuracy: 0.8011\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4781 - sparse_categorical_accuracy: 0.8107 - val_loss: 0.3526 - val_sparse_categorical_accuracy: 0.8795\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4090 - sparse_categorical_accuracy: 0.8420 - val_loss: 0.3073 - val_sparse_categorical_accuracy: 0.8911\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3641 - sparse_categorical_accuracy: 0.8667 - val_loss: 0.2761 - val_sparse_categorical_accuracy: 0.9008\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3319 - sparse_categorical_accuracy: 0.8792 - val_loss: 0.2491 - val_sparse_categorical_accuracy: 0.9120\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2997 - sparse_categorical_accuracy: 0.8939 - val_loss: 0.2514 - val_sparse_categorical_accuracy: 0.9161\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2750 - sparse_categorical_accuracy: 0.9032 - val_loss: 0.2157 - val_sparse_categorical_accuracy: 0.9232\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2614 - sparse_categorical_accuracy: 0.9083 - val_loss: 0.2037 - val_sparse_categorical_accuracy: 0.9242\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2448 - sparse_categorical_accuracy: 0.9125 - val_loss: 0.2014 - val_sparse_categorical_accuracy: 0.9268\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2389 - sparse_categorical_accuracy: 0.9163 - val_loss: 0.1873 - val_sparse_categorical_accuracy: 0.9354\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2226 - sparse_categorical_accuracy: 0.9262 - val_loss: 0.1870 - val_sparse_categorical_accuracy: 0.9298\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2087 - sparse_categorical_accuracy: 0.9263 - val_loss: 0.1807 - val_sparse_categorical_accuracy: 0.9354\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2041 - sparse_categorical_accuracy: 0.9304 - val_loss: 0.1747 - val_sparse_categorical_accuracy: 0.9364\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1972 - sparse_categorical_accuracy: 0.9308 - val_loss: 0.1771 - val_sparse_categorical_accuracy: 0.9451\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1938 - sparse_categorical_accuracy: 0.9335 - val_loss: 0.1936 - val_sparse_categorical_accuracy: 0.9329\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1844 - sparse_categorical_accuracy: 0.9374 - val_loss: 0.1650 - val_sparse_categorical_accuracy: 0.9420\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1849 - sparse_categorical_accuracy: 0.9369 - val_loss: 0.1779 - val_sparse_categorical_accuracy: 0.9379\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1800 - sparse_categorical_accuracy: 0.9386 - val_loss: 0.1669 - val_sparse_categorical_accuracy: 0.9461\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1688 - sparse_categorical_accuracy: 0.9422 - val_loss: 0.1658 - val_sparse_categorical_accuracy: 0.9364\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1704 - sparse_categorical_accuracy: 0.9440 - val_loss: 0.1583 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1685 - sparse_categorical_accuracy: 0.9436 - val_loss: 0.1654 - val_sparse_categorical_accuracy: 0.9369\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1620 - sparse_categorical_accuracy: 0.9462 - val_loss: 0.1624 - val_sparse_categorical_accuracy: 0.9435\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1501 - sparse_categorical_accuracy: 0.9492 - val_loss: 0.1634 - val_sparse_categorical_accuracy: 0.9430\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1516 - sparse_categorical_accuracy: 0.9482 - val_loss: 0.1641 - val_sparse_categorical_accuracy: 0.9405\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1497 - sparse_categorical_accuracy: 0.9503 - val_loss: 0.1608 - val_sparse_categorical_accuracy: 0.9440\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00026: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 4ms/step - loss: 0.1583 - sparse_categorical_accuracy: 0.9420\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4608 - sparse_categorical_accuracy: 0.8563\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.3976 - sparse_categorical_accuracy: 0.8647\n",
            "Load dataset with random state 3\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30824008 0.3809766  0.31078332]%\n",
            "train P, S and noise labels is [0.31026587 0.38010431 0.30962982]%\n",
            "compile the model\n",
            "Fit model on training data step 4\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 3s 7ms/step - loss: 1.0580 - sparse_categorical_accuracy: 0.5894 - val_loss: 1.1984 - val_sparse_categorical_accuracy: 0.5692\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.6352 - sparse_categorical_accuracy: 0.7402 - val_loss: 0.6794 - val_sparse_categorical_accuracy: 0.6963\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4942 - sparse_categorical_accuracy: 0.8050 - val_loss: 0.3592 - val_sparse_categorical_accuracy: 0.8784\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4077 - sparse_categorical_accuracy: 0.8477 - val_loss: 0.2933 - val_sparse_categorical_accuracy: 0.9013\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3657 - sparse_categorical_accuracy: 0.8683 - val_loss: 0.2535 - val_sparse_categorical_accuracy: 0.9166\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3290 - sparse_categorical_accuracy: 0.8836 - val_loss: 0.2535 - val_sparse_categorical_accuracy: 0.9044\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3119 - sparse_categorical_accuracy: 0.8900 - val_loss: 0.2197 - val_sparse_categorical_accuracy: 0.9318\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2809 - sparse_categorical_accuracy: 0.9017 - val_loss: 0.2018 - val_sparse_categorical_accuracy: 0.9303\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2610 - sparse_categorical_accuracy: 0.9107 - val_loss: 0.1929 - val_sparse_categorical_accuracy: 0.9410\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2556 - sparse_categorical_accuracy: 0.9112 - val_loss: 0.1819 - val_sparse_categorical_accuracy: 0.9415\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2400 - sparse_categorical_accuracy: 0.9213 - val_loss: 0.1769 - val_sparse_categorical_accuracy: 0.9476\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2306 - sparse_categorical_accuracy: 0.9209 - val_loss: 0.1669 - val_sparse_categorical_accuracy: 0.9481\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2186 - sparse_categorical_accuracy: 0.9274 - val_loss: 0.1680 - val_sparse_categorical_accuracy: 0.9446\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2209 - sparse_categorical_accuracy: 0.9238 - val_loss: 0.1722 - val_sparse_categorical_accuracy: 0.9390\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2019 - sparse_categorical_accuracy: 0.9291 - val_loss: 0.1545 - val_sparse_categorical_accuracy: 0.9517\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1882 - sparse_categorical_accuracy: 0.9347 - val_loss: 0.1503 - val_sparse_categorical_accuracy: 0.9502\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1864 - sparse_categorical_accuracy: 0.9345 - val_loss: 0.1600 - val_sparse_categorical_accuracy: 0.9471\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1859 - sparse_categorical_accuracy: 0.9352 - val_loss: 0.1587 - val_sparse_categorical_accuracy: 0.9435\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1840 - sparse_categorical_accuracy: 0.9393 - val_loss: 0.1469 - val_sparse_categorical_accuracy: 0.9502\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1756 - sparse_categorical_accuracy: 0.9397 - val_loss: 0.1468 - val_sparse_categorical_accuracy: 0.9532\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1785 - sparse_categorical_accuracy: 0.9364 - val_loss: 0.1386 - val_sparse_categorical_accuracy: 0.9552\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1666 - sparse_categorical_accuracy: 0.9466 - val_loss: 0.1462 - val_sparse_categorical_accuracy: 0.9537\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1603 - sparse_categorical_accuracy: 0.9438 - val_loss: 0.1381 - val_sparse_categorical_accuracy: 0.9552\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1575 - sparse_categorical_accuracy: 0.9506 - val_loss: 0.1427 - val_sparse_categorical_accuracy: 0.9517\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1571 - sparse_categorical_accuracy: 0.9487 - val_loss: 0.1535 - val_sparse_categorical_accuracy: 0.9486\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1460 - sparse_categorical_accuracy: 0.9490 - val_loss: 0.1381 - val_sparse_categorical_accuracy: 0.9517\n",
            "Epoch 27/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1460 - sparse_categorical_accuracy: 0.9505 - val_loss: 0.1613 - val_sparse_categorical_accuracy: 0.9471\n",
            "Epoch 28/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1483 - sparse_categorical_accuracy: 0.9467 - val_loss: 0.1383 - val_sparse_categorical_accuracy: 0.9532\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00028: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 4ms/step - loss: 0.1381 - sparse_categorical_accuracy: 0.9552\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4084 - sparse_categorical_accuracy: 0.8639\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 312s 32ms/step - loss: 0.4329 - sparse_categorical_accuracy: 0.8497\n",
            "Load dataset with random state 4\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30773143 0.36978637 0.3224822 ]%\n",
            "train P, S and noise labels is [0.31039308 0.38290294 0.30670398]%\n",
            "compile the model\n",
            "Fit model on training data step 5\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 3s 7ms/step - loss: 0.9976 - sparse_categorical_accuracy: 0.6125 - val_loss: 1.0136 - val_sparse_categorical_accuracy: 0.5824\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.5821 - sparse_categorical_accuracy: 0.7705 - val_loss: 0.5440 - val_sparse_categorical_accuracy: 0.8449\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4661 - sparse_categorical_accuracy: 0.8157 - val_loss: 0.3151 - val_sparse_categorical_accuracy: 0.8922\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4116 - sparse_categorical_accuracy: 0.8419 - val_loss: 0.2784 - val_sparse_categorical_accuracy: 0.9105\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3682 - sparse_categorical_accuracy: 0.8668 - val_loss: 0.2456 - val_sparse_categorical_accuracy: 0.9196\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3394 - sparse_categorical_accuracy: 0.8779 - val_loss: 0.2328 - val_sparse_categorical_accuracy: 0.9201\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3073 - sparse_categorical_accuracy: 0.8895 - val_loss: 0.2081 - val_sparse_categorical_accuracy: 0.9313\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2945 - sparse_categorical_accuracy: 0.8970 - val_loss: 0.1986 - val_sparse_categorical_accuracy: 0.9395\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2778 - sparse_categorical_accuracy: 0.9033 - val_loss: 0.1856 - val_sparse_categorical_accuracy: 0.9456\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2646 - sparse_categorical_accuracy: 0.9106 - val_loss: 0.1731 - val_sparse_categorical_accuracy: 0.9496\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2475 - sparse_categorical_accuracy: 0.9139 - val_loss: 0.1656 - val_sparse_categorical_accuracy: 0.9456\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2339 - sparse_categorical_accuracy: 0.9166 - val_loss: 0.1640 - val_sparse_categorical_accuracy: 0.9502\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2207 - sparse_categorical_accuracy: 0.9247 - val_loss: 0.1564 - val_sparse_categorical_accuracy: 0.9517\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2204 - sparse_categorical_accuracy: 0.9242 - val_loss: 0.1586 - val_sparse_categorical_accuracy: 0.9491\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2078 - sparse_categorical_accuracy: 0.9283 - val_loss: 0.1466 - val_sparse_categorical_accuracy: 0.9583\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1942 - sparse_categorical_accuracy: 0.9342 - val_loss: 0.1527 - val_sparse_categorical_accuracy: 0.9588\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1918 - sparse_categorical_accuracy: 0.9332 - val_loss: 0.1473 - val_sparse_categorical_accuracy: 0.9568\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1881 - sparse_categorical_accuracy: 0.9355 - val_loss: 0.1342 - val_sparse_categorical_accuracy: 0.9608\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1840 - sparse_categorical_accuracy: 0.9377 - val_loss: 0.1366 - val_sparse_categorical_accuracy: 0.9557\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1707 - sparse_categorical_accuracy: 0.9401 - val_loss: 0.1415 - val_sparse_categorical_accuracy: 0.9552\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1710 - sparse_categorical_accuracy: 0.9410 - val_loss: 0.1436 - val_sparse_categorical_accuracy: 0.9588\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1654 - sparse_categorical_accuracy: 0.9457 - val_loss: 0.1334 - val_sparse_categorical_accuracy: 0.9603\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1549 - sparse_categorical_accuracy: 0.9459 - val_loss: 0.1440 - val_sparse_categorical_accuracy: 0.9552\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1518 - sparse_categorical_accuracy: 0.9450 - val_loss: 0.1356 - val_sparse_categorical_accuracy: 0.9547\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1490 - sparse_categorical_accuracy: 0.9484 - val_loss: 0.1286 - val_sparse_categorical_accuracy: 0.9649\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1424 - sparse_categorical_accuracy: 0.9487 - val_loss: 0.1337 - val_sparse_categorical_accuracy: 0.9619\n",
            "Epoch 27/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1428 - sparse_categorical_accuracy: 0.9533 - val_loss: 0.1374 - val_sparse_categorical_accuracy: 0.9593\n",
            "Epoch 28/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1304 - sparse_categorical_accuracy: 0.9552 - val_loss: 0.1422 - val_sparse_categorical_accuracy: 0.9532\n",
            "Epoch 29/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1340 - sparse_categorical_accuracy: 0.9529 - val_loss: 0.1316 - val_sparse_categorical_accuracy: 0.9573\n",
            "Epoch 30/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1281 - sparse_categorical_accuracy: 0.9550 - val_loss: 0.1284 - val_sparse_categorical_accuracy: 0.9629\n",
            "Epoch 31/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1195 - sparse_categorical_accuracy: 0.9589 - val_loss: 0.1294 - val_sparse_categorical_accuracy: 0.9619\n",
            "Epoch 32/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1162 - sparse_categorical_accuracy: 0.9594 - val_loss: 0.1333 - val_sparse_categorical_accuracy: 0.9603\n",
            "Epoch 33/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1137 - sparse_categorical_accuracy: 0.9588 - val_loss: 0.1331 - val_sparse_categorical_accuracy: 0.9639\n",
            "Epoch 34/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1045 - sparse_categorical_accuracy: 0.9608 - val_loss: 0.1322 - val_sparse_categorical_accuracy: 0.9644\n",
            "Epoch 35/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1020 - sparse_categorical_accuracy: 0.9654 - val_loss: 0.1827 - val_sparse_categorical_accuracy: 0.9502\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00035: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 4ms/step - loss: 0.1284 - sparse_categorical_accuracy: 0.9629\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4477 - sparse_categorical_accuracy: 0.8640\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 312s 32ms/step - loss: 0.5048 - sparse_categorical_accuracy: 0.8396\n",
            "Load dataset with random state 5\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30315361 0.37436419 0.3224822 ]%\n",
            "train P, S and noise labels is [0.31153797 0.38175805 0.30670398]%\n",
            "compile the model\n",
            "Fit model on training data step 6\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 3s 7ms/step - loss: 1.0154 - sparse_categorical_accuracy: 0.5943 - val_loss: 1.0366 - val_sparse_categorical_accuracy: 0.3749\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.5913 - sparse_categorical_accuracy: 0.7642 - val_loss: 0.7069 - val_sparse_categorical_accuracy: 0.5931\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4735 - sparse_categorical_accuracy: 0.8117 - val_loss: 0.3120 - val_sparse_categorical_accuracy: 0.8917\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4171 - sparse_categorical_accuracy: 0.8415 - val_loss: 0.2781 - val_sparse_categorical_accuracy: 0.9140\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3697 - sparse_categorical_accuracy: 0.8605 - val_loss: 0.2404 - val_sparse_categorical_accuracy: 0.9115\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3419 - sparse_categorical_accuracy: 0.8766 - val_loss: 0.2243 - val_sparse_categorical_accuracy: 0.9329\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3072 - sparse_categorical_accuracy: 0.8903 - val_loss: 0.2171 - val_sparse_categorical_accuracy: 0.9268\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2959 - sparse_categorical_accuracy: 0.8965 - val_loss: 0.2011 - val_sparse_categorical_accuracy: 0.9385\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2608 - sparse_categorical_accuracy: 0.9052 - val_loss: 0.1781 - val_sparse_categorical_accuracy: 0.9446\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2612 - sparse_categorical_accuracy: 0.9117 - val_loss: 0.1727 - val_sparse_categorical_accuracy: 0.9471\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2375 - sparse_categorical_accuracy: 0.9139 - val_loss: 0.1715 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2355 - sparse_categorical_accuracy: 0.9210 - val_loss: 0.1631 - val_sparse_categorical_accuracy: 0.9496\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2191 - sparse_categorical_accuracy: 0.9242 - val_loss: 0.1641 - val_sparse_categorical_accuracy: 0.9466\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2124 - sparse_categorical_accuracy: 0.9275 - val_loss: 0.1519 - val_sparse_categorical_accuracy: 0.9481\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2041 - sparse_categorical_accuracy: 0.9288 - val_loss: 0.1511 - val_sparse_categorical_accuracy: 0.9522\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2017 - sparse_categorical_accuracy: 0.9317 - val_loss: 0.1543 - val_sparse_categorical_accuracy: 0.9522\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1876 - sparse_categorical_accuracy: 0.9370 - val_loss: 0.1476 - val_sparse_categorical_accuracy: 0.9542\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1819 - sparse_categorical_accuracy: 0.9391 - val_loss: 0.1410 - val_sparse_categorical_accuracy: 0.9527\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1754 - sparse_categorical_accuracy: 0.9405 - val_loss: 0.1429 - val_sparse_categorical_accuracy: 0.9532\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1697 - sparse_categorical_accuracy: 0.9439 - val_loss: 0.1365 - val_sparse_categorical_accuracy: 0.9537\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1614 - sparse_categorical_accuracy: 0.9468 - val_loss: 0.1500 - val_sparse_categorical_accuracy: 0.9486\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1574 - sparse_categorical_accuracy: 0.9471 - val_loss: 0.1340 - val_sparse_categorical_accuracy: 0.9578\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1586 - sparse_categorical_accuracy: 0.9462 - val_loss: 0.1354 - val_sparse_categorical_accuracy: 0.9532\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1464 - sparse_categorical_accuracy: 0.9490 - val_loss: 0.1405 - val_sparse_categorical_accuracy: 0.9512\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1418 - sparse_categorical_accuracy: 0.9524 - val_loss: 0.1762 - val_sparse_categorical_accuracy: 0.9420\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1444 - sparse_categorical_accuracy: 0.9487 - val_loss: 0.1585 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 27/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1361 - sparse_categorical_accuracy: 0.9533 - val_loss: 0.1697 - val_sparse_categorical_accuracy: 0.9476\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00027: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 4ms/step - loss: 0.1340 - sparse_categorical_accuracy: 0.9578\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.3893 - sparse_categorical_accuracy: 0.8667\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 312s 32ms/step - loss: 0.4043 - sparse_categorical_accuracy: 0.8617\n",
            "Load dataset with random state 6\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30620549 0.37945066 0.31434385]%\n",
            "train P, S and noise labels is [0.31077471 0.38048594 0.30873935]%\n",
            "compile the model\n",
            "Fit model on training data step 7\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 3s 7ms/step - loss: 0.9699 - sparse_categorical_accuracy: 0.6227 - val_loss: 1.0617 - val_sparse_categorical_accuracy: 0.3103\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.5497 - sparse_categorical_accuracy: 0.7827 - val_loss: 0.5279 - val_sparse_categorical_accuracy: 0.8561\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4419 - sparse_categorical_accuracy: 0.8255 - val_loss: 0.3334 - val_sparse_categorical_accuracy: 0.8927\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3807 - sparse_categorical_accuracy: 0.8589 - val_loss: 0.2877 - val_sparse_categorical_accuracy: 0.9039\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3524 - sparse_categorical_accuracy: 0.8685 - val_loss: 0.2634 - val_sparse_categorical_accuracy: 0.9125\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3158 - sparse_categorical_accuracy: 0.8841 - val_loss: 0.2428 - val_sparse_categorical_accuracy: 0.9201\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2968 - sparse_categorical_accuracy: 0.8928 - val_loss: 0.2302 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2729 - sparse_categorical_accuracy: 0.9008 - val_loss: 0.2330 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2557 - sparse_categorical_accuracy: 0.9066 - val_loss: 0.2262 - val_sparse_categorical_accuracy: 0.9293\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2482 - sparse_categorical_accuracy: 0.9155 - val_loss: 0.2021 - val_sparse_categorical_accuracy: 0.9359\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2358 - sparse_categorical_accuracy: 0.9166 - val_loss: 0.2011 - val_sparse_categorical_accuracy: 0.9420\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2254 - sparse_categorical_accuracy: 0.9218 - val_loss: 0.1912 - val_sparse_categorical_accuracy: 0.9359\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2098 - sparse_categorical_accuracy: 0.9263 - val_loss: 0.1915 - val_sparse_categorical_accuracy: 0.9415\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2051 - sparse_categorical_accuracy: 0.9290 - val_loss: 0.1974 - val_sparse_categorical_accuracy: 0.9379\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2027 - sparse_categorical_accuracy: 0.9304 - val_loss: 0.1909 - val_sparse_categorical_accuracy: 0.9379\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1892 - sparse_categorical_accuracy: 0.9354 - val_loss: 0.1832 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1900 - sparse_categorical_accuracy: 0.9340 - val_loss: 0.1778 - val_sparse_categorical_accuracy: 0.9430\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1819 - sparse_categorical_accuracy: 0.9375 - val_loss: 0.1823 - val_sparse_categorical_accuracy: 0.9410\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1710 - sparse_categorical_accuracy: 0.9405 - val_loss: 0.1769 - val_sparse_categorical_accuracy: 0.9435\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1774 - sparse_categorical_accuracy: 0.9394 - val_loss: 0.1718 - val_sparse_categorical_accuracy: 0.9471\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1629 - sparse_categorical_accuracy: 0.9467 - val_loss: 0.1748 - val_sparse_categorical_accuracy: 0.9420\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1601 - sparse_categorical_accuracy: 0.9487 - val_loss: 0.1780 - val_sparse_categorical_accuracy: 0.9461\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1578 - sparse_categorical_accuracy: 0.9452 - val_loss: 0.1740 - val_sparse_categorical_accuracy: 0.9435\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1574 - sparse_categorical_accuracy: 0.9480 - val_loss: 0.1647 - val_sparse_categorical_accuracy: 0.9481\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1484 - sparse_categorical_accuracy: 0.9504 - val_loss: 0.1873 - val_sparse_categorical_accuracy: 0.9410\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1523 - sparse_categorical_accuracy: 0.9515 - val_loss: 0.1729 - val_sparse_categorical_accuracy: 0.9430\n",
            "Epoch 27/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1419 - sparse_categorical_accuracy: 0.9536 - val_loss: 0.1650 - val_sparse_categorical_accuracy: 0.9446\n",
            "Epoch 28/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1366 - sparse_categorical_accuracy: 0.9537 - val_loss: 0.1884 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 29/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1247 - sparse_categorical_accuracy: 0.9564 - val_loss: 0.1714 - val_sparse_categorical_accuracy: 0.9471\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00029: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 4ms/step - loss: 0.1647 - sparse_categorical_accuracy: 0.9481\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.3993 - sparse_categorical_accuracy: 0.8682\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 313s 32ms/step - loss: 0.4867 - sparse_categorical_accuracy: 0.8443\n",
            "Load dataset with random state 7\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.29857579 0.3819939  0.31943032]%\n",
            "train P, S and noise labels is [0.31268286 0.37984989 0.30746724]%\n",
            "compile the model\n",
            "Fit model on training data step 8\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 3s 7ms/step - loss: 0.9328 - sparse_categorical_accuracy: 0.6316 - val_loss: 1.0343 - val_sparse_categorical_accuracy: 0.3174\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.5417 - sparse_categorical_accuracy: 0.7836 - val_loss: 0.5491 - val_sparse_categorical_accuracy: 0.8469\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4404 - sparse_categorical_accuracy: 0.8292 - val_loss: 0.3237 - val_sparse_categorical_accuracy: 0.8952\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3864 - sparse_categorical_accuracy: 0.8515 - val_loss: 0.2932 - val_sparse_categorical_accuracy: 0.9049\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3566 - sparse_categorical_accuracy: 0.8683 - val_loss: 0.2572 - val_sparse_categorical_accuracy: 0.9166\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3211 - sparse_categorical_accuracy: 0.8849 - val_loss: 0.2449 - val_sparse_categorical_accuracy: 0.9130\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2955 - sparse_categorical_accuracy: 0.8917 - val_loss: 0.2249 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2767 - sparse_categorical_accuracy: 0.8982 - val_loss: 0.2093 - val_sparse_categorical_accuracy: 0.9273\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2652 - sparse_categorical_accuracy: 0.9084 - val_loss: 0.1953 - val_sparse_categorical_accuracy: 0.9334\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2473 - sparse_categorical_accuracy: 0.9134 - val_loss: 0.2023 - val_sparse_categorical_accuracy: 0.9415\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2311 - sparse_categorical_accuracy: 0.9229 - val_loss: 0.1835 - val_sparse_categorical_accuracy: 0.9410\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2314 - sparse_categorical_accuracy: 0.9223 - val_loss: 0.1972 - val_sparse_categorical_accuracy: 0.9435\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2190 - sparse_categorical_accuracy: 0.9255 - val_loss: 0.1837 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2033 - sparse_categorical_accuracy: 0.9314 - val_loss: 0.1711 - val_sparse_categorical_accuracy: 0.9456\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2002 - sparse_categorical_accuracy: 0.9346 - val_loss: 0.1921 - val_sparse_categorical_accuracy: 0.9364\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1969 - sparse_categorical_accuracy: 0.9327 - val_loss: 0.1693 - val_sparse_categorical_accuracy: 0.9456\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2022 - sparse_categorical_accuracy: 0.9354 - val_loss: 0.1655 - val_sparse_categorical_accuracy: 0.9461\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1841 - sparse_categorical_accuracy: 0.9356 - val_loss: 0.1680 - val_sparse_categorical_accuracy: 0.9486\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1731 - sparse_categorical_accuracy: 0.9377 - val_loss: 0.1658 - val_sparse_categorical_accuracy: 0.9507\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1715 - sparse_categorical_accuracy: 0.9422 - val_loss: 0.1554 - val_sparse_categorical_accuracy: 0.9517\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1673 - sparse_categorical_accuracy: 0.9438 - val_loss: 0.1641 - val_sparse_categorical_accuracy: 0.9502\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1639 - sparse_categorical_accuracy: 0.9453 - val_loss: 0.1534 - val_sparse_categorical_accuracy: 0.9481\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1546 - sparse_categorical_accuracy: 0.9492 - val_loss: 0.1549 - val_sparse_categorical_accuracy: 0.9496\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1483 - sparse_categorical_accuracy: 0.9503 - val_loss: 0.1701 - val_sparse_categorical_accuracy: 0.9415\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1416 - sparse_categorical_accuracy: 0.9505 - val_loss: 0.1753 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1424 - sparse_categorical_accuracy: 0.9522 - val_loss: 0.1556 - val_sparse_categorical_accuracy: 0.9527\n",
            "Epoch 27/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1396 - sparse_categorical_accuracy: 0.9534 - val_loss: 0.1548 - val_sparse_categorical_accuracy: 0.9507\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00027: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 4ms/step - loss: 0.1534 - sparse_categorical_accuracy: 0.9481\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4110 - sparse_categorical_accuracy: 0.8589\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 312s 32ms/step - loss: 0.4347 - sparse_categorical_accuracy: 0.8507\n",
            "Load dataset with random state 8\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.30417091 0.38046796 0.31536114]%\n",
            "train P, S and noise labels is [0.31128355 0.38023152 0.30848493]%\n",
            "compile the model\n",
            "Fit model on training data step 9\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 3s 7ms/step - loss: 0.9584 - sparse_categorical_accuracy: 0.6274 - val_loss: 1.0573 - val_sparse_categorical_accuracy: 0.3042\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.5385 - sparse_categorical_accuracy: 0.7867 - val_loss: 0.5284 - val_sparse_categorical_accuracy: 0.8189\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4361 - sparse_categorical_accuracy: 0.8270 - val_loss: 0.3208 - val_sparse_categorical_accuracy: 0.8825\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3900 - sparse_categorical_accuracy: 0.8504 - val_loss: 0.2837 - val_sparse_categorical_accuracy: 0.9013\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3466 - sparse_categorical_accuracy: 0.8742 - val_loss: 0.2711 - val_sparse_categorical_accuracy: 0.8973\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3246 - sparse_categorical_accuracy: 0.8781 - val_loss: 0.2428 - val_sparse_categorical_accuracy: 0.9176\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2965 - sparse_categorical_accuracy: 0.8940 - val_loss: 0.2354 - val_sparse_categorical_accuracy: 0.9196\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2617 - sparse_categorical_accuracy: 0.9073 - val_loss: 0.2069 - val_sparse_categorical_accuracy: 0.9298\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2553 - sparse_categorical_accuracy: 0.9082 - val_loss: 0.2015 - val_sparse_categorical_accuracy: 0.9364\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2429 - sparse_categorical_accuracy: 0.9155 - val_loss: 0.1943 - val_sparse_categorical_accuracy: 0.9395\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2382 - sparse_categorical_accuracy: 0.9168 - val_loss: 0.1876 - val_sparse_categorical_accuracy: 0.9395\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2167 - sparse_categorical_accuracy: 0.9261 - val_loss: 0.1845 - val_sparse_categorical_accuracy: 0.9364\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2012 - sparse_categorical_accuracy: 0.9305 - val_loss: 0.1807 - val_sparse_categorical_accuracy: 0.9400\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1941 - sparse_categorical_accuracy: 0.9345 - val_loss: 0.1790 - val_sparse_categorical_accuracy: 0.9405\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1983 - sparse_categorical_accuracy: 0.9313 - val_loss: 0.1731 - val_sparse_categorical_accuracy: 0.9400\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1898 - sparse_categorical_accuracy: 0.9360 - val_loss: 0.1736 - val_sparse_categorical_accuracy: 0.9405\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1819 - sparse_categorical_accuracy: 0.9392 - val_loss: 0.1900 - val_sparse_categorical_accuracy: 0.9410\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1784 - sparse_categorical_accuracy: 0.9408 - val_loss: 0.1716 - val_sparse_categorical_accuracy: 0.9440\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1712 - sparse_categorical_accuracy: 0.9397 - val_loss: 0.1654 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1697 - sparse_categorical_accuracy: 0.9410 - val_loss: 0.1694 - val_sparse_categorical_accuracy: 0.9446\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1656 - sparse_categorical_accuracy: 0.9420 - val_loss: 0.1684 - val_sparse_categorical_accuracy: 0.9430\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1602 - sparse_categorical_accuracy: 0.9484 - val_loss: 0.1649 - val_sparse_categorical_accuracy: 0.9430\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1588 - sparse_categorical_accuracy: 0.9443 - val_loss: 0.1592 - val_sparse_categorical_accuracy: 0.9461\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1548 - sparse_categorical_accuracy: 0.9490 - val_loss: 0.1682 - val_sparse_categorical_accuracy: 0.9451\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1334 - sparse_categorical_accuracy: 0.9531 - val_loss: 0.1682 - val_sparse_categorical_accuracy: 0.9466\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1405 - sparse_categorical_accuracy: 0.9531 - val_loss: 0.1634 - val_sparse_categorical_accuracy: 0.9461\n",
            "Epoch 27/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1423 - sparse_categorical_accuracy: 0.9514 - val_loss: 0.1656 - val_sparse_categorical_accuracy: 0.9486\n",
            "Epoch 28/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1285 - sparse_categorical_accuracy: 0.9566 - val_loss: 0.1768 - val_sparse_categorical_accuracy: 0.9446\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00028: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 4ms/step - loss: 0.1590 - sparse_categorical_accuracy: 0.9461\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4401 - sparse_categorical_accuracy: 0.8620\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 312s 32ms/step - loss: 0.4673 - sparse_categorical_accuracy: 0.8477\n",
            "Load dataset with random state 9\n",
            "Total samples 9827\n",
            "P 3045, S 3737, Noise 3045\n",
            "test P, S and noise labels is [0.31892167 0.37487284 0.30620549]%\n",
            "train P, S and noise labels is [0.30759445 0.38163084 0.31077471]%\n",
            "compile the model\n",
            "Fit model on training data step 10\n",
            "Epoch 1/400\n",
            "246/246 [==============================] - 3s 7ms/step - loss: 0.9869 - sparse_categorical_accuracy: 0.6219 - val_loss: 1.0673 - val_sparse_categorical_accuracy: 0.3204\n",
            "Epoch 2/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.5742 - sparse_categorical_accuracy: 0.7670 - val_loss: 0.5070 - val_sparse_categorical_accuracy: 0.8474\n",
            "Epoch 3/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4623 - sparse_categorical_accuracy: 0.8199 - val_loss: 0.3170 - val_sparse_categorical_accuracy: 0.9013\n",
            "Epoch 4/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.4063 - sparse_categorical_accuracy: 0.8444 - val_loss: 0.2757 - val_sparse_categorical_accuracy: 0.9125\n",
            "Epoch 5/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3612 - sparse_categorical_accuracy: 0.8634 - val_loss: 0.2576 - val_sparse_categorical_accuracy: 0.9237\n",
            "Epoch 6/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3365 - sparse_categorical_accuracy: 0.8793 - val_loss: 0.2256 - val_sparse_categorical_accuracy: 0.9308\n",
            "Epoch 7/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.3101 - sparse_categorical_accuracy: 0.8888 - val_loss: 0.2102 - val_sparse_categorical_accuracy: 0.9283\n",
            "Epoch 8/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2772 - sparse_categorical_accuracy: 0.9033 - val_loss: 0.2026 - val_sparse_categorical_accuracy: 0.9369\n",
            "Epoch 9/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2631 - sparse_categorical_accuracy: 0.9092 - val_loss: 0.1861 - val_sparse_categorical_accuracy: 0.9364\n",
            "Epoch 10/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2473 - sparse_categorical_accuracy: 0.9141 - val_loss: 0.1865 - val_sparse_categorical_accuracy: 0.9334\n",
            "Epoch 11/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2461 - sparse_categorical_accuracy: 0.9173 - val_loss: 0.1853 - val_sparse_categorical_accuracy: 0.9405\n",
            "Epoch 12/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2296 - sparse_categorical_accuracy: 0.9192 - val_loss: 0.1687 - val_sparse_categorical_accuracy: 0.9446\n",
            "Epoch 13/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2203 - sparse_categorical_accuracy: 0.9260 - val_loss: 0.1654 - val_sparse_categorical_accuracy: 0.9461\n",
            "Epoch 14/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2094 - sparse_categorical_accuracy: 0.9291 - val_loss: 0.1755 - val_sparse_categorical_accuracy: 0.9481\n",
            "Epoch 15/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2000 - sparse_categorical_accuracy: 0.9333 - val_loss: 0.1557 - val_sparse_categorical_accuracy: 0.9502\n",
            "Epoch 16/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.2060 - sparse_categorical_accuracy: 0.9293 - val_loss: 0.1676 - val_sparse_categorical_accuracy: 0.9425\n",
            "Epoch 17/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1913 - sparse_categorical_accuracy: 0.9359 - val_loss: 0.1585 - val_sparse_categorical_accuracy: 0.9481\n",
            "Epoch 18/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1847 - sparse_categorical_accuracy: 0.9370 - val_loss: 0.1498 - val_sparse_categorical_accuracy: 0.9537\n",
            "Epoch 19/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1816 - sparse_categorical_accuracy: 0.9368 - val_loss: 0.1505 - val_sparse_categorical_accuracy: 0.9512\n",
            "Epoch 20/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1754 - sparse_categorical_accuracy: 0.9401 - val_loss: 0.1475 - val_sparse_categorical_accuracy: 0.9522\n",
            "Epoch 21/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1709 - sparse_categorical_accuracy: 0.9415 - val_loss: 0.1474 - val_sparse_categorical_accuracy: 0.9491\n",
            "Epoch 22/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1637 - sparse_categorical_accuracy: 0.9452 - val_loss: 0.1451 - val_sparse_categorical_accuracy: 0.9522\n",
            "Epoch 23/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1608 - sparse_categorical_accuracy: 0.9468 - val_loss: 0.1670 - val_sparse_categorical_accuracy: 0.9451\n",
            "Epoch 24/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1577 - sparse_categorical_accuracy: 0.9478 - val_loss: 0.1505 - val_sparse_categorical_accuracy: 0.9512\n",
            "Epoch 25/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1463 - sparse_categorical_accuracy: 0.9510 - val_loss: 0.1410 - val_sparse_categorical_accuracy: 0.9517\n",
            "Epoch 26/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1502 - sparse_categorical_accuracy: 0.9481 - val_loss: 0.1432 - val_sparse_categorical_accuracy: 0.9552\n",
            "Epoch 27/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1529 - sparse_categorical_accuracy: 0.9486 - val_loss: 0.1413 - val_sparse_categorical_accuracy: 0.9532\n",
            "Epoch 28/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1371 - sparse_categorical_accuracy: 0.9552 - val_loss: 0.1441 - val_sparse_categorical_accuracy: 0.9547\n",
            "Epoch 29/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1310 - sparse_categorical_accuracy: 0.9534 - val_loss: 0.1385 - val_sparse_categorical_accuracy: 0.9547\n",
            "Epoch 30/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1311 - sparse_categorical_accuracy: 0.9559 - val_loss: 0.1482 - val_sparse_categorical_accuracy: 0.9502\n",
            "Epoch 31/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1302 - sparse_categorical_accuracy: 0.9552 - val_loss: 0.1432 - val_sparse_categorical_accuracy: 0.9537\n",
            "Epoch 32/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1244 - sparse_categorical_accuracy: 0.9573 - val_loss: 0.1488 - val_sparse_categorical_accuracy: 0.9512\n",
            "Epoch 33/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1238 - sparse_categorical_accuracy: 0.9567 - val_loss: 0.1463 - val_sparse_categorical_accuracy: 0.9532\n",
            "Epoch 34/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1207 - sparse_categorical_accuracy: 0.9602 - val_loss: 0.1368 - val_sparse_categorical_accuracy: 0.9547\n",
            "Epoch 35/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1141 - sparse_categorical_accuracy: 0.9608 - val_loss: 0.1408 - val_sparse_categorical_accuracy: 0.9512\n",
            "Epoch 36/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1166 - sparse_categorical_accuracy: 0.9608 - val_loss: 0.1347 - val_sparse_categorical_accuracy: 0.9547\n",
            "Epoch 37/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1024 - sparse_categorical_accuracy: 0.9645 - val_loss: 0.1517 - val_sparse_categorical_accuracy: 0.9527\n",
            "Epoch 38/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.0977 - sparse_categorical_accuracy: 0.9658 - val_loss: 0.1461 - val_sparse_categorical_accuracy: 0.9537\n",
            "Epoch 39/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.1014 - sparse_categorical_accuracy: 0.9648 - val_loss: 0.1374 - val_sparse_categorical_accuracy: 0.9552\n",
            "Epoch 40/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.0949 - sparse_categorical_accuracy: 0.9668 - val_loss: 0.1460 - val_sparse_categorical_accuracy: 0.9502\n",
            "Epoch 41/400\n",
            "246/246 [==============================] - 1s 6ms/step - loss: 0.0910 - sparse_categorical_accuracy: 0.9690 - val_loss: 0.1475 - val_sparse_categorical_accuracy: 0.9537\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00041: early stopping\n",
            "Evaluate transformer based model on SAKHALIN test data\n",
            "16/16 [==============================] - 0s 4ms/step - loss: 0.1343 - sparse_categorical_accuracy: 0.9547\n",
            "Evaluate transformer based model on DAGESTAN test data\n",
            "218/218 [==============================] - 1s 4ms/step - loss: 0.4245 - sparse_categorical_accuracy: 0.8760\n",
            "Evaluate transformer based model on CALIFORNIA test data\n",
            "9846/9846 [==============================] - 312s 32ms/step - loss: 0.4797 - sparse_categorical_accuracy: 0.8559\n",
            "CPU times: user 25min 25s, sys: 5min 54s, total: 31min 19s\n",
            "Wall time: 1h 44s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kw09-uHqTJqk"
      },
      "source": [
        "### Print collected values and scores\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AsIMJ73FTJqk",
        "outputId": "4df6f8f0-fd44-4049-dc16-c7b299091d06"
      },
      "source": [
        "print(val_sparse_categorical_accuracy)\n",
        "print(val_loss)\n",
        "print(sakh_top_1)\n",
        "print(dag_top_1)\n",
        "print(cal_top_1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.2899287939071655, 0.7039674520492554, 0.8692777156829834, 0.8875890374183655, 0.904374361038208, 0.9079349040985107, 0.9211597442626953, 0.9211597442626953, 0.9267548322677612, 0.928280770778656, 0.9308239817619324, 0.9394710063934326, 0.9409969449043274, 0.938453733921051, 0.9333672523498535, 0.9425228834152222, 0.9430315494537354, 0.9425228834152222, 0.9440488219261169, 0.9460834264755249, 0.9476093649864197, 0.9338759183883667, 0.9465920925140381, 0.9491353034973145, 0.9506612420082092, 0.9476093649864197, 0.9516785144805908, 0.950152575969696, 0.9511699080467224, 0.9476093649864197, 0.9440488219261169, 0.9526958465576172, 0.9450660943984985], [0.4038657248020172, 0.8138352036476135, 0.8926754593849182, 0.9094608426094055, 0.9170905351638794, 0.9160732626914978, 0.9298067092895508, 0.928280770778656, 0.9323499202728271, 0.9354018568992615, 0.9389623403549194, 0.9425228834152222, 0.9496439695358276, 0.9460834264755249, 0.9455747604370117, 0.9465920925140381, 0.9511699080467224, 0.9476093649864197, 0.9537131190299988, 0.9481180310249329, 0.9511699080467224, 0.9425228834152222, 0.9526958465576172, 0.9455747604370117, 0.9476093649864197, 0.9496439695358276, 0.9471006989479065, 0.9491353034973145], [0.3702950179576874, 0.8011190295219421, 0.8794506788253784, 0.8911495208740234, 0.9008138179779053, 0.9120040535926819, 0.9160732626914978, 0.9231942892074585, 0.9242116212844849, 0.9267548322677612, 0.9354018568992615, 0.9298067092895508, 0.9354018568992615, 0.9364191293716431, 0.9450660943984985, 0.9328585863113403, 0.942014217376709, 0.9379450678825378, 0.9460834264755249, 0.9364191293716431, 0.9425228834152222, 0.9369277954101562, 0.9435401558876038, 0.9430315494537354, 0.9404882788658142, 0.9440488219261169], [0.5691760182380676, 0.6963377594947815, 0.878433346748352, 0.9013224840164185, 0.9165818691253662, 0.904374361038208, 0.9318413138389587, 0.930315375328064, 0.9409969449043274, 0.9415056109428406, 0.9476093649864197, 0.9481180310249329, 0.9445574879646301, 0.9389623403549194, 0.9516785144805908, 0.950152575969696, 0.9471006989479065, 0.9435401558876038, 0.950152575969696, 0.9532044529914856, 0.9552390575408936, 0.9537131190299988, 0.9552390575408936, 0.9516785144805908, 0.9486266374588013, 0.9516785144805908, 0.9471006989479065, 0.9532044529914856], [0.5824007987976074, 0.8448626399040222, 0.8921668529510498, 0.9104781150817871, 0.9196337461471558, 0.920142412185669, 0.9313326478004456, 0.9394710063934326, 0.9455747604370117, 0.9496439695358276, 0.9455747604370117, 0.950152575969696, 0.9516785144805908, 0.9491353034973145, 0.9582909345626831, 0.9587996006011963, 0.9567649960517883, 0.9608342051506042, 0.9557477235794067, 0.9552390575408936, 0.9587996006011963, 0.9603255391120911, 0.9552390575408936, 0.9547303915023804, 0.9649033546447754, 0.9618514776229858, 0.9593082666397095, 0.9532044529914856, 0.9572736620903015, 0.9628687500953674, 0.9618514776229858, 0.9603255391120911, 0.9638860821723938, 0.9643946886062622, 0.950152575969696], [0.3748728334903717, 0.5930824279785156, 0.8916581869125366, 0.9140386581420898, 0.9114954471588135, 0.9328585863113403, 0.9267548322677612, 0.938453733921051, 0.9445574879646301, 0.9471006989479065, 0.9425228834152222, 0.9496439695358276, 0.9465920925140381, 0.9481180310249329, 0.952187180519104, 0.952187180519104, 0.954221785068512, 0.9526958465576172, 0.9532044529914856, 0.9537131190299988, 0.9486266374588013, 0.9577823281288147, 0.9532044529914856, 0.9511699080467224, 0.942014217376709, 0.9440488219261169, 0.9476093649864197], [0.3102746605873108, 0.8560528755187988, 0.8926754593849182, 0.9038656949996948, 0.9125127196311951, 0.920142412185669, 0.928280770778656, 0.928280770778656, 0.9292980432510376, 0.9359104633331299, 0.942014217376709, 0.9359104633331299, 0.9415056109428406, 0.9379450678825378, 0.9379450678825378, 0.9440488219261169, 0.9430315494537354, 0.9409969449043274, 0.9435401558876038, 0.9471006989479065, 0.942014217376709, 0.9460834264755249, 0.9435401558876038, 0.9481180310249329, 0.9409969449043274, 0.9430315494537354, 0.9445574879646301, 0.9440488219261169, 0.9471006989479065], [0.31739571690559387, 0.8468972444534302, 0.8952187299728394, 0.9048830270767212, 0.9165818691253662, 0.9130213856697083, 0.928280770778656, 0.9272634983062744, 0.9333672523498535, 0.9415056109428406, 0.9409969449043274, 0.9435401558876038, 0.9440488219261169, 0.9455747604370117, 0.9364191293716431, 0.9455747604370117, 0.9460834264755249, 0.9486266374588013, 0.9506612420082092, 0.9516785144805908, 0.950152575969696, 0.9481180310249329, 0.9496439695358276, 0.9415056109428406, 0.9440488219261169, 0.9526958465576172, 0.9506612420082092], [0.3041709065437317, 0.818921685218811, 0.882502555847168, 0.9013224840164185, 0.8972533345222473, 0.9175992012023926, 0.9196337461471558, 0.9298067092895508, 0.9364191293716431, 0.9394710063934326, 0.9394710063934326, 0.9364191293716431, 0.9399796724319458, 0.9404882788658142, 0.9399796724319458, 0.9404882788658142, 0.9409969449043274, 0.9440488219261169, 0.9425228834152222, 0.9445574879646301, 0.9430315494537354, 0.9430315494537354, 0.9460834264755249, 0.9450660943984985, 0.9465920925140381, 0.9460834264755249, 0.9486266374588013, 0.9445574879646301], [0.3204476237297058, 0.8474059104919434, 0.9013224840164185, 0.9125127196311951, 0.9237029552459717, 0.9308239817619324, 0.928280770778656, 0.9369277954101562, 0.9364191293716431, 0.9333672523498535, 0.9404882788658142, 0.9445574879646301, 0.9460834264755249, 0.9481180310249329, 0.950152575969696, 0.9425228834152222, 0.9481180310249329, 0.9537131190299988, 0.9511699080467224, 0.952187180519104, 0.9491353034973145, 0.952187180519104, 0.9450660943984985, 0.9511699080467224, 0.9516785144805908, 0.9552390575408936, 0.9532044529914856, 0.9547303915023804, 0.9547303915023804, 0.950152575969696, 0.9537131190299988, 0.9511699080467224, 0.9532044529914856, 0.9547303915023804, 0.9511699080467224, 0.9547303915023804, 0.9526958465576172, 0.9537131190299988, 0.9552390575408936, 0.950152575969696, 0.9537131190299988]]\n",
            "[[1.481429934501648, 0.7337744832038879, 0.3683016002178192, 0.3135067820549011, 0.28642797470092773, 0.26261594891548157, 0.24246147274971008, 0.22760318219661713, 0.21883068978786469, 0.20622661709785461, 0.208321675658226, 0.19888868927955627, 0.19532261788845062, 0.18738499283790588, 0.2007146179676056, 0.1766018122434616, 0.17444296181201935, 0.17197103798389435, 0.1722879409790039, 0.16955426335334778, 0.16782039403915405, 0.21095691621303558, 0.17133168876171112, 0.16335828602313995, 0.1673290878534317, 0.1777358055114746, 0.16350573301315308, 0.16232630610466003, 0.16250576078891754, 0.16770878434181213, 0.17430949211120605, 0.16232699155807495, 0.17453941702842712], [0.9766126275062561, 0.5015183091163635, 0.3364354372024536, 0.28696149587631226, 0.26002782583236694, 0.2488546073436737, 0.23161177337169647, 0.20774270594120026, 0.2070222795009613, 0.19714407622814178, 0.1881134957075119, 0.18297675251960754, 0.1764189749956131, 0.17445148527622223, 0.1701286882162094, 0.16630031168460846, 0.16658331453800201, 0.16801238059997559, 0.16140036284923553, 0.16473516821861267, 0.15617243945598602, 0.1688690036535263, 0.15471811592578888, 0.1611020416021347, 0.16135013103485107, 0.16725051403045654, 0.1739887297153473, 0.15772467851638794], [1.022268295288086, 0.5420175194740295, 0.3526375889778137, 0.3072945475578308, 0.2761141359806061, 0.24907557666301727, 0.25137612223625183, 0.21574784815311432, 0.20373541116714478, 0.2014210969209671, 0.1873038113117218, 0.18703794479370117, 0.18066218495368958, 0.17467962205410004, 0.17707981169223785, 0.19357383251190186, 0.1649545282125473, 0.17787984013557434, 0.16689780354499817, 0.16582532227039337, 0.15833327174186707, 0.1653614342212677, 0.16239865124225616, 0.1633962094783783, 0.1641267091035843, 0.16080880165100098], [1.19839346408844, 0.6793690323829651, 0.35915589332580566, 0.2933349311351776, 0.25353798270225525, 0.25349077582359314, 0.21968211233615875, 0.20180070400238037, 0.1928911656141281, 0.1818637102842331, 0.17686711251735687, 0.1668609082698822, 0.16796541213989258, 0.17216789722442627, 0.15449561178684235, 0.15033026039600372, 0.15995515882968903, 0.1587245911359787, 0.14692386984825134, 0.1467711478471756, 0.1385720670223236, 0.1461949646472931, 0.13806775212287903, 0.14267738163471222, 0.15351176261901855, 0.1381174474954605, 0.16126404702663422, 0.13834302127361298], [1.0135942697525024, 0.5439829230308533, 0.31509390473365784, 0.27840298414230347, 0.24563173949718475, 0.23282913863658905, 0.20806050300598145, 0.19855642318725586, 0.18562592566013336, 0.17307986319065094, 0.16563060879707336, 0.16404682397842407, 0.15641486644744873, 0.1586034744977951, 0.14663387835025787, 0.15274976193904877, 0.14727576076984406, 0.13419675827026367, 0.13664613664150238, 0.141537606716156, 0.14364103972911835, 0.1333947777748108, 0.14397136867046356, 0.13558262586593628, 0.12860551476478577, 0.1336551010608673, 0.13742338120937347, 0.1421758234500885, 0.13158369064331055, 0.12835055589675903, 0.1293536275625229, 0.1332666575908661, 0.1330789178609848, 0.1322326511144638, 0.18273243308067322], [1.0366398096084595, 0.7069239020347595, 0.3120041489601135, 0.2781427502632141, 0.24042412638664246, 0.22425737977027893, 0.21714217960834503, 0.20109912753105164, 0.17813898622989655, 0.1726958006620407, 0.17150835692882538, 0.16314876079559326, 0.1640864908695221, 0.15185661613941193, 0.15111452341079712, 0.1542617529630661, 0.14758993685245514, 0.14097625017166138, 0.14286716282367706, 0.13646262884140015, 0.14998874068260193, 0.13399171829223633, 0.13542389869689941, 0.14054584503173828, 0.1762460172176361, 0.15847617387771606, 0.16968099772930145], [1.0617024898529053, 0.5279492735862732, 0.3334161937236786, 0.2877366542816162, 0.2634386718273163, 0.24277998507022858, 0.230156809091568, 0.23295919597148895, 0.2262009084224701, 0.20207509398460388, 0.20107126235961914, 0.19118808209896088, 0.19147706031799316, 0.19741572439670563, 0.19094787538051605, 0.18316461145877838, 0.1777871996164322, 0.18232089281082153, 0.17686687409877777, 0.17184962332248688, 0.17481884360313416, 0.17797265946865082, 0.1740221083164215, 0.16470734775066376, 0.1872888207435608, 0.17285040020942688, 0.16497686505317688, 0.188372403383255, 0.1713745892047882], [1.0342705249786377, 0.5491414070129395, 0.32370659708976746, 0.29322096705436707, 0.25715312361717224, 0.24486736953258514, 0.22492220997810364, 0.20931175351142883, 0.19529667496681213, 0.20234280824661255, 0.18345320224761963, 0.19720450043678284, 0.1837153285741806, 0.17111949622631073, 0.19205574691295624, 0.16928623616695404, 0.16545440256595612, 0.16797304153442383, 0.16576291620731354, 0.1553887277841568, 0.16413289308547974, 0.15335224568843842, 0.15494109690189362, 0.17009925842285156, 0.17527244985103607, 0.15562477707862854, 0.15476170182228088], [1.057289481163025, 0.5284019112586975, 0.3208191990852356, 0.28372910618782043, 0.2710892856121063, 0.24275703728199005, 0.23537583649158478, 0.2069242298603058, 0.2014646828174591, 0.1942984163761139, 0.18763130903244019, 0.18451344966888428, 0.1807154268026352, 0.17900991439819336, 0.17313550412654877, 0.17360493540763855, 0.18999311327934265, 0.1715765744447708, 0.16540566086769104, 0.1693958342075348, 0.16835111379623413, 0.16490046679973602, 0.1591770499944687, 0.16816328465938568, 0.16824641823768616, 0.16342294216156006, 0.16555456817150116, 0.17680977284908295], [1.067342758178711, 0.5069839954376221, 0.31698930263519287, 0.27567827701568604, 0.25759992003440857, 0.22556255757808685, 0.21023552119731903, 0.20256434381008148, 0.18607114255428314, 0.18651126325130463, 0.18532021343708038, 0.16873322427272797, 0.16544124484062195, 0.17550110816955566, 0.1557082086801529, 0.16763602197170258, 0.15849624574184418, 0.14981436729431152, 0.15048855543136597, 0.1474558413028717, 0.14742963016033173, 0.14512421190738678, 0.1669645756483078, 0.1504795253276825, 0.1410118192434311, 0.14322234690189362, 0.1413470208644867, 0.14413709938526154, 0.1385340839624405, 0.14819106459617615, 0.1431831419467926, 0.14876993000507355, 0.14632929861545563, 0.13680794835090637, 0.14084698259830475, 0.13469848036766052, 0.15172633528709412, 0.14607475697994232, 0.13737896084785461, 0.14598038792610168, 0.14746209979057312]]\n",
            "[0.950152575969696, 0.9526958465576172, 0.942014217376709, 0.9552390575408936, 0.9628687500953674, 0.9577823281288147, 0.9481180310249329, 0.9481180310249329, 0.9460834264755249, 0.9547303915023804]\n",
            "[0.8670499324798584, 0.8604024648666382, 0.8563061356544495, 0.8638879060745239, 0.8639956712722778, 0.8667265772819519, 0.8681997656822205, 0.8588573336601257, 0.8619834780693054, 0.8760330677032471]\n",
            "[0.8517709970474243, 0.8392910957336426, 0.8646931648254395, 0.8496690988540649, 0.8395732045173645, 0.8617150187492371, 0.8442921042442322, 0.8506631255149841, 0.8477345108985901, 0.8559398055076599]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TAr4LoFXTJql",
        "outputId": "a1632f36-14e7-44bf-828b-d273e1dc205d"
      },
      "source": [
        "print(\"Mean SAKHALIN test TOP-1 accuracy {}, std. dev {}\".format(np.mean(sakh_top_1), np.std(sakh_top_1)))\n",
        "print(\"Mean DAGESTAN test TOP-1 accuracy {}, std. dev {}\".format(np.mean(dag_top_1), np.std(dag_top_1)))\n",
        "print(\"Mean CALIFIRNIA test TOP-1 accuracy {}, std. dev {}\".format(np.mean(cal_top_1), np.std(cal_top_1)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mean SAKHALIN test TOP-1 accuracy 0.9517802655696869, std. dev 0.005811944490719707\n",
            "Mean DAGESTAN test TOP-1 accuracy 0.8643442332744599, std. dev 0.005297620839961549\n",
            "Mean CALIFIRNIA test TOP-1 accuracy 0.8505342125892639, std. dev 0.008064459600339498\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jAhDXNmSDR4w"
      },
      "source": [
        "# Plot Section"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bmMNA_R8fqPm",
        "outputId": "558e108f-60b7-40df-aff2-e0f2fb333faa"
      },
      "source": [
        "model_with_spec_cal = seismo_performer_with_spec(\n",
        "                                        maxlen=400,\n",
        "                                        nfft=64,\n",
        "                                        hop_length=16,\n",
        "                                        patch_size_1=22,\n",
        "                                        patch_size_2=3,\n",
        "                                        num_channels=3,\n",
        "                                        num_patches=11,\n",
        "                                        d_model=48,\n",
        "                                        num_heads=2,\n",
        "                                        ff_dim_factor=2,\n",
        "                                        layers_depth=2,\n",
        "                                        num_classes=3,\n",
        "                                        drop_out_rate=0.1)\n",
        "\n",
        "LR = 0.0001\n",
        "model_with_spec_cal.compile(\n",
        "    optimizer=keras.optimizers.Adam(learning_rate=LR),\n",
        "    loss=keras.losses.SparseCategoricalCrossentropy(),\n",
        "    metrics=[keras.metrics.SparseCategoricalAccuracy()],)\n",
        "\n",
        "model_with_spec_cal.summary()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_4\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_5 (InputLayer)         [(None, 400, 3)]          0         \n",
            "_________________________________________________________________\n",
            "stft_3 (STFT)                (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude_3 (Magnitude)      (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "magnitude_to_decibel_3 (Magn (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "max_abs_scaler_3 (MaxABSScal (None, 22, 33, 3)         0         \n",
            "_________________________________________________________________\n",
            "rearrange3d_2 (Rearrange3d)  (None, 11, 198)           0         \n",
            "_________________________________________________________________\n",
            "dense_21 (Dense)             (None, 11, 48)            9552      \n",
            "_________________________________________________________________\n",
            "cls_token_2 (ClsToken)       (None, 12, 48)            48        \n",
            "_________________________________________________________________\n",
            "pos_embeding2_2 (PosEmbeding (None, 12, 48)            576       \n",
            "_________________________________________________________________\n",
            "performer_block_4 (Performer (None, 12, 48)            18672     \n",
            "_________________________________________________________________\n",
            "performer_block_5 (Performer (None, 12, 48)            18672     \n",
            "_________________________________________________________________\n",
            "lambda_2 (Lambda)            (None, 48)                0         \n",
            "_________________________________________________________________\n",
            "layer_normalization_14 (Laye (None, 48)                96        \n",
            "_________________________________________________________________\n",
            "dropout_20 (Dropout)         (None, 48)                0         \n",
            "_________________________________________________________________\n",
            "dense_26 (Dense)             (None, 96)                4704      \n",
            "_________________________________________________________________\n",
            "dropout_21 (Dropout)         (None, 96)                0         \n",
            "_________________________________________________________________\n",
            "dense_27 (Dense)             (None, 48)                4656      \n",
            "_________________________________________________________________\n",
            "dropout_22 (Dropout)         (None, 48)                0         \n",
            "_________________________________________________________________\n",
            "dense_28 (Dense)             (None, 3)                 147       \n",
            "=================================================================\n",
            "Total params: 57,123\n",
            "Trainable params: 57,123\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zbTaxhRQDU7p"
      },
      "source": [
        "extractor = keras.Model(inputs=model_with_spec_cal.inputs,\n",
        "                        outputs=[layer.output for layer in model_with_spec_cal.layers])"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kTKYd9Gpnb4z",
        "outputId": "d9c51521-d6ec-422b-920a-0bc3e4d9208f"
      },
      "source": [
        "y_test_sakh[7005]"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WrUhEDNeDd_X"
      },
      "source": [
        "features = extractor(np.expand_dims(X_test_sakh[7005], axis=0), tf.convert_to_tensor(y_test_sakh[7005]))"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MdEsA3s1jdpI"
      },
      "source": [
        "### input layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 513
        },
        "id": "9NrDT9HZDube",
        "outputId": "8bc67893-d80f-4746-89f9-08f9064ada8f"
      },
      "source": [
        "for i in range(3):\n",
        "  ax = plt.subplot(3, 1, i + 1)\n",
        "  plt.plot(features[0].numpy()[0][:,i], 'black',  linewidth=1)\n",
        "  #ax.figsize=(3, 22)\n",
        "  #ax.imshow(patches[i]).set_clim(cmin, cmax)\n",
        "  #im = ax.matshow(features[3].numpy()[0][:,:,0])\n",
        "  #patch = mpl.patches.Rectangle((0+i*3,0),3, 22, transform=ax.transData)\n",
        "  #im.set_clip_path(patch)\n",
        "  plt.axis(\"off\")\n",
        "\n",
        "plt.show()\n",
        "\n",
        "# Z-ch only\n",
        "plt.plot(features[0].numpy()[0][:,2])"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAADnCAYAAAC9roUQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd7xdZZX3vys9IdRAek9IAUMoAaKA9CKio6CjY0PHsTvqjIXX8dXXPpZXx8FRZl7b2MaOODAivRMgdNJIL6STRkIKgaz3j7U299xz9zln79PvZX0/n/O59+zy7Ofsc/Zvr72etdYjqkoQBEHQHHq1ugNBEAQvJUJ0gyAImkiIbhAEQRMJ0Q2CIGgiIbpBEARNJEQ3CIKgiYToBkEQNJEQ3SBoU0Skn4gcKyJ9Wt2XoH6E6AZB+/Ju4D5gi4i8vNWdqQUxDm91P9qBEN0gaF8uAy4HPgb8c4v7UisvBx5odSfagRDdIGhDROQQ4FTgBuDnwEnd3FI8EZgsIhNb3ZFWE6IbBO3JscAiVX1WVZ8H5gEzWtynWjgO2AecV68GReQ4ERlZr/aaRYhuELQnxwALCt4/QZ1EV0R6i0j/erSVg+OAXwGvrGOb3wE+Usf2mkKIbhC0J8cC8wveP04dRFdE+gIrgWdE5EciIrW2meGYArwM+DEwq05tHgycDlxQj/aaSYhuELQnxaI7H7N+a+Uk4GngcMzPelkd2qzEcGA3MAcYLSKH1qHNs4B7gQkiMrQO7TWNEN0gaDPcMjwBs24TVgDj69D82cDtqrob+AlwYR3arMQkYJn7ph/DhL9WpgMPAo9grotuQ4huELQfU4HdqrqmYNlTwDB3D9TCWcDt/v9tmAg3mknAMv//XuC0OrQ5HnOTLAKm1aG9phGiGwTtxxnAnYUL3ErcAIyutlEX7JcDd/mi+cChIjKm2jYzMhFY7v/fSX0G08bTIbrT69Be0wjRDYL24xzgjpTlK6nNxTALe8zfCqCqBzCrt9HWbqGlexcwW0T61djmeGAVYekGQVALbo1eCPw5ZfUqYFwNzb8duKVoWTNcDCdgccao6nZgKzCq2sbc5z2ODtF9WZ4ojBaEy3UiRDcI2ovTMWt0fcq6lVRp6YrI6cDFwFeLVjVUdN11MQx4tGDxZuDIGpo9Etinqs8Aa4AdZBycE5G3ANtEpGWDbyG6QdBevAa4tsS6WizddwNXJq6FAhYB/UVkQpXtvognXZxctPh84GZVfaFg2dPUJrpjsXOB2nTmfwDekHHfD2C+7DfVcPyaCNENgjbBH5FfS2nRXUkVlq6Xhnw98F/F61y0bqdGa1dEPoGFtd0pIn9fsOocuro0ngaOquFwo7FojoSfAO+udOMQkV7ATOAzwKU1HL8mQnSDoH04H7smHy2xvlpLdyKwRVU3lFhfDxfDW4BPAWcC74cXbyJn0hGillCrpTsGcysAoKqLge8Bn66w3yRgG/Z5J4jIwBr6UDUhukHQBrhAfR/4gFufaawBRolI75zNTwcWlll/G3B2tSnBXhFtCvBH4GFgrIgcBkwAegNLi3apVXSLLV2A/wDeKCKDy+x3PPCIqu7Hnhom19CHqgnRDYL2YDxwEHBjqQ1UdR8mWHkra03HfLelSESxWhGaDTykqvs8nvgh4BTMyr0j5SZSD9EtTBzBBx7vAN5cZr+j6TgPi7AbRdMJ0Q2COiAiA2osHnMGcFcZKzdhJfn9umUtXT/mHd6HaphU1P693tZZdHUtQO3RC2PoaukC/AB4X5nvYSyw2v9/Esv8azohukFQBSIySkSuFJGTfABnEfDlGpo8Dbg7w3bV+HWnUt7SxddXa+kOBTYVvL8BC087k/Qkj0a4FwD+AggwV0Q+k7I+RDcIujHfwB5X/xu72H+KWVnVhnRNoXP93FKsJL+lO4GONNxSLPftqqFYdO/FBHwjJm7FVC26HoEwihTR9bC0S4HfAP+QEs1QKLrLsAHG4vZHVNOvPIToBkFOvDThq7EMr7cCVwKfxyy8aqt2FQpCOVaRQ3RFZBBwCCaA5VhBighlpJPo+kDVO4HXlXCX1BIydiSwS1X3pK1U1dWq+k3gJgoK6xRlsYHdZDp9XhE5B3hKRI6usm+ZCNENgvy8FhsgelpVb1fV77m43EAVRbXdeusyOFSCleRzL4wHVnmdhXJ0EaEcFFu6qOofS2TVgaUBH15FFAYUhYuVYTGdB8oOAw6o6g5/vw4YUhQ29hUs+uKDVfQrMyG6QZCf1wNXpyy/HjhHRGaJyBE52hsObC9lvRWRy9LFXAYrM2y3ERjkMzLkpYvolsMt4Z2YEOallD+3mCV0Ft1OTxLuiliNn0sPe3sZ8HfAZY2cUSNENwhy4PGn5wLXFa9T1c3AL7DyhfNFZHjGZl9Ma83AKmCMW8dZGI+5DsrilvpKqvPr5hJdp1q/bqnIhWIWYz73hHF0dd+soOPznoYVRX8ciy2eVEXfMtGnUQ1nRURGYXek7dgPZBrQDxiBjUReg8Uu9lbV5zK2OQAbmTwEi30cgGWiPAY85+s2AYOANaq6N6WNIdiXth97PHvaA69PA5ar6pKCbQdi6YW9gTmlHuX8QhkHKHan35ohRCjxRx2OFfZQrLjHm7E5s7YDv1DV/87QTh/sAhkK7FTVZQXrRmLFVnYBtxVaXT64sKkof764f8mPdAyW174a+IOqpg2kFO8/CfsuDscsmcXYdz8Ki0nd6p/9KWBejnM2ydt7Ffadb8d+E/2wKl4/TvvuK/Bh4BpV3VJi/f8CvgZ8C/P7/ihDm2mCkIqq7hGRHZh1vC7DLpPIILpO4mJ4vNKGCV4V7RDsO8pDEjZW8fdRRFY3zBJgioiI/17SbmxLsd/Fn7Eb6R2qqiJyq78vTuqoCw0RXR9o+BkdlvQcrMjECmA9JoKvAS6nI8bvCGAtJox76bjjfBXLRVcR2YaFoCz39p7EBHsEVj5uJDAYi0tcjgntbmCPr5sGDMQu6iF+nKEi8iQwFwvqPoAVz5iNnfQ+wHgR2e9tPwhM8h/+MuyOfQF2AQzAfFWKidcaP/bB/poAPOvHOBR4WkTu88+R/DCGeP/mYfNknYHNjdUXE6Z92AV6NfBt7OL7hoh83T/PXj+HgomrYDeaYdhNRLAf/FAROeBt9ccu/DuxR75fici93vdj/BiIyCI//lzs4pzp536Af3f7ff0PsRvobSKyFStIciMdIjoSS3ndi114g/18bfH+TAGe9zbX+ecY5N/rTu/bQuAZ/z43Yhf+id5Wf+zmKNiFdhfwS/8Otvvyy4ErROQLwM9U9XkRGVBOhEXkAmz22ZIzH/g0OLtF5EbgIrKJ7niyuQASVmLfVxbRnUp6rGwa1UQwHImlGFfyGRdTraU7mjIJJAmquk1E9mK/3fWkD1Q+CJzvxtAbgUt8+RzqM6VQKo2ydPdiM38ewMTilcArsC80eeS6HvgCcEMpC8r5rte/3I9d4OdhF+3rMatqKXZx3oKJ4F7gMf/xd6Hgzpe8H4jNsXQycCp2Ef8IuExVd/k2fbDR1o2qesDv7tP88xwFfFNV57l1NQF4AROJMZgg7fTXGlXdmPQDS0s8DrswFBP9Lb78NdhN6v8AC1V1vQ88DC4YDEg+wy8xQeqDic7xfr42+XcwEJt1YJ2qLvJ9evv5HOvNPODZRIjIkZjV2xsTrYexC2Syt38K9gRyhbe7O+07FJGPYjevy7Dpsjf4d7Qbq/bUB7Ng52e5aP3iOBUT1+nelz2YVbIHuxls98/+JT9vpaziP4jIK7Cb+pdFZAMwQ0QexUS6P1aP4CAs6H64f47LCp9yynAT8M3i31sJJmBTrGdlJSbUczJsWykFuJDl5I/Vrca1ALW5F7JYutDhYliP3aSKa1rcB/xvTJ92qmryHSzAakk0BMnwpBYEPRa/+U3EBGABNp3NTEy478FuoH+L3Qy/p6qZBUZE1gBnFbpxSmx3A/CvqppWuDxt+29grqmvVdhuIPbYf3ByQ62w/WuB96nqq1PW9QP+HZvU8mcFy88HrlDV87L0vegzbFHVr+fcbxlwUZYbn4j8BLhHVX8oInOAT6rq3QXre2HnZxnwfVX9kS8fiiWLDMniyspLy326QdBK/KJaRsd0MjfS9fH14Sqbn4s9QZUVXTrPIZaFlWSbAfdobPyhouA65dwLZwHvwtxUPytYXoulm2vq9ILEiLUZdykMGxtPkXvBn1o/irkHf1awfJOIvIB91lKV2aomoheCoHE8gLlDSuJunjHk8+lmTQU+iXxui2XAOE+oKGYKNt15sVBWK7ob6XA1ZiVJjEh1HaawBDhaRMZirrIubglV/amqvtXD2ApZgI1n1J0Q3SBoHPdg9QfKMRJzFeSJongSG2StxDnArVkb9YiVeZh1XszRWPJHcQJFtaJbHEebhazhYgkPY2NJ5wF35nQVvBPz+dadEN0gaBxzsGLZ5Sy6vK4FMIv0IA/zS8V91efSddaGStyDCVUxR2Ofp6/HKidUK7pPAtNyJiFkDRcDQFWXY77Zz1M0pX2GfVfksKhzEaIbBA3Cfak3Aa8rs9kEssfRJu0qZoWVc12Mxq7vvIJ+A/BBESl+tD4as06X0zlxoCrR9TjnfeRzMYwlh+g6XwR+ixUkagtCdIOgsXwb+FyZtODcouvcT3nRnQYsyDv6rqo3YCFyH02WeVLQSExwV9A5DXkoFvddDYu8n1k5hsolKjuhqreo6ieKwyxbSYhuEDQQVb0PmxDy1yUepatxL4D5XqeXWT+N/NleCb8BLi7o71nA/T5zxXo6W6fDqM69ABaXnqdA0LHY5+7WhOgGQeO5AnskPyFlXbWW7mLKF+GeRk6rsKjt/XSM3l+AuR3AQqiSDMUkwSZrCFcx3wbeKyIVq3r5DeBYLBO1WxOiGwQNxrP1fonV3y2mWtFdiqWn9y2xPstsEam4S+IeYLaIHAT8NfAnX/2i6GJW7rYq6lckx1mNpd5+TkRKRmOIyN8A78CyNqu1qtuGEN0gaA6/Bt5Q6GLwokoHky8MCgAXunWULvM4kcpJGeVIEjvei4VbJQJeKLpZC6+XRFXnYJluqdauiMwE/tX78YVGZIg1mxDdIGgOC7FiR7MKlp0APFpFsZiE4kLdQPkpbXKQiO7lwFUFy+squs6fML9xGh8Hvqaqp6nqd+twrJYTohsETcAttD9ihZoSTqT6FGOwzLSxKcuPAp6p9rHfeZSOMqCFMa6FA2n1Et15mKukUwF1z4x7LTYQ2WMI0Q2C5vGi6HoBmVdRm+iuxSzaYrLOrlASz06bBby+qILcJqwsaC/qJLqegvsoXcspXgg8pKp1r3/QSkJ0g6B5PAgcLCKnAzdjZS1/X0N7T2ECW0zNogugqhtV9f6iZfuwMqVHYn7jlbUex3kAKxlayGvpGMDrMYToBkGTcN/tlViW2lLg0hpTTUtZunlqzlZDUnBnKtXHAhczlwLR9XC0V2MTGPQoorRjEDSXf8VmB/liDQNoCQ1zL1RgBSa4Y6gtQqKQB7BpjhJOBTaoajXhdG1NWLpB0ERUdZ+qXqGqz9ahuVLuhawzAFfLSqyC2Zqs8xZmYBkw2OfjA5s5peK8f92REN0g6L7sAHr59OGFJMVpGsVKbJCrXq6FJLrjXmyaKLBBxkwzaXQ3QnSDoJviQpXM4Au8mC47mQbNZOuswArg5CqXmIHbgbN8YtvJ2MBjjyNENwi6N8XFwIcCz6nqtgYecz5WlvHKOrd7G+a2eDkwt46ui7YiBtKCoHtTnJU2mca6FlDVldgs1/XmUWyG508AdzSg/bYgLN0g6N4Ui+5UX9bt8CSMH2GRC1dV2LzbEqIbBN2b4hKPM4HHW9SXevAv2BTrG1vdkUYRohsE3Zv5wLEikrgKj6Mbi66q7lDVe1rdj0YSohsE3RhV3Y4lSUz3yIWZwGOt7VVQjhhIC4LuT1KGcQcWudDtC333ZEJ0g6D78wAwGytEc3+FbYMWE6IbBN2f24CPAbsI0W17wqcbBN2feVjc7BuB+1rcl6ACYekGQTdHVVVEfoJV/erRI/89AekB87wFQRB0G8K9EARB0ERCdIMgCJpIiG4QBEETCdENgiBoIiG6QRAETSRENwiCoImE6AZBEDSREN0gCIImEqIbBEHQREJ0gyAImkiIbhAEQRMJ0Q2CIGgiIbpBEARNJEQ3CIKgiYToBkEQNJEQ3SAIgiYSohsEPQgRGS0inxaR0a3uS5BOTNcTBD0EEekF/BIY5q+PtbZHQRph6QZBz+EkYDjwTuDc1nYlHRH5uoic0+p+tJIQ3SDoOZwHXA/MBUaLyLAW9yeNS4F3tboTrSRENwh6DucCt6jqC8D9wCkt7k8nRORgbMbiS0SkX6v70ypCdIOgByAiAswC5viiR4HjW9ejVGYCjwFrgWNb3JeWEaIbBD2Do4AXVPVpf/8IcEIL+5PGcdjN4AlgRov70jJCdIOgZzAFWFzwvh0t3bHASkJ0gyDoARSL7lLgKBE5rEX9SWMksI4Gi66IHCUirxeRvo06Ri2E6AZBz6CT6Ppg2hOYH7VdSER3HvCyBh7nX4CfA5c38BhVE6IbBD2DYksXzMXQTn7dRHRXA0NE5KB6H0BE+gOvBj4DXFZDO68TkfF16lYnQnSDoGeQJrrtNpg2CljrVvhSYHIDjnEmsAD4MfBKERlQZTtfAQ6vW68KCNENgm6OiPQGJgFLilYtAKY1v0ddEZHBQF9ghy9aDExtwKFmA3er6k5gDXZeciEih2ODfk/UuW9AiG4Q9ATGAptVdXfR8mXAxBb0J40RwDpVVX+/GLPO683JwAP+/xLg6CramA3MVdXn69arAkJ0g6D7k+ZaANgIDBKRQ5rcnzQSf25CtYJYEk8QOYXOoluNsM/CMvoaQohuEHR/UkXXrcrltIe1OwrLREtYDdS7/ORwTNOe8vfVCvsE7CmhIYToBkELEJFJIvJlEfmKiLyhxpHyUpYutI/oFlu6a7A6DACIyBkickyNxzgGmF/gwqhWdMcDq2rsS0lCdIOgyYjIdOA+bGDpBeDdwM0iMrDKJiuJbjWDSX1E5Aqv0VsPikX3KawSmojICOAa4FciUkuN72OwwcOEVZi/Oy/jCNENgh7Fu4AfqOoVqvo5VX0VJhZvrbK9cqJb7WP88cDXgLOq7FMxnURXVZ8F9gJDsHjaa4EDwCtqOEax6K4FRrqvNxMeCTIaO28NIUQ3CJqIC8Cbgf8qWnUdcEYV7Q3AIgNWlthkLdWJ7hnANswKrwcj6ezTBXMxjAZOxKqjzSVHpppbyecWlIk8FliYrPdojt2YsGdlBLBVVffm2CcXIbpB0FxmAvtUdV7R8ruB06tobwqwokx401PYIFZeTge+DFxcpxoGo+jsXoAOv+6JwMPAfPKlB38BK9r+EXeDHI9l4RWylnyfv6GuBQjRDYJmczEmFMUsAg5z/2YeZgKPl1mfV3QSjgFuxDLHXnzkF5Ej/RE8M27djwTWF61ahvV/CpaIkLcmw18D7wA+jRXQ2aKqW4q2yWvpp1nkdSVENwiahGc6XY75LzuhqgeAhzCrLw8z6WrdFbIeGJZXKLGJLddjE13+XER2i8jtmCC9o3hjf9S/SURmpbR1BLDX/biFPAB8EHjCH+fnk7G4uYiMwmoI/xZLd/48dv6KyWvpjwA25Ng+NyG6QdAERORVmABcC9xcYrOHyS+6aY/UL6KqzwFbMRHNhBeNGQxsU9XvAG/CxP0bWCGZ16TsdjpW9+CLKetG0RE7W8h9mMgllv9G4OCM9RJeCdzpN6sfAq/z9opJtfT9JnGq3wgLGU5Xi7yuhOgGQYNxf+M3gDer6icK4kiLeZgcBWq83RMob+mCh2dlbRcYCmxyQUNV56jqElX9M/BToHDwKuEtmOCenSKao0l/ZF8ObMJF18/LJrLdIF6Gf25V/TVmTX87ZbtS7pW/xUT6vUXLRxCiGwTNp4rH8XKcAAzAIhTKkXe2hxnYSHslkViJBfxnZThmdXZBVTdjYllskc8GbvJ1xYVsUi1dF9kTVLUw5XYD2UR3Op0jFbYlN4kiSvl0LwNupWuls3AvBEEz8VkHvgHsEJHVIvIdEfmgiIypuHNpLgauK2PhJqwARuWYKfdc4JYM2+XNShtGeeG5BzgteeNJHVOxSSfT/LKlLF1UtTiiYSPZRLc4JrcUXXy6XvHsDODf6Jqx1nD3Qi3ZHyXxx55xwAZV3eMFN6YD5wCHAU8DNwCKfVlnYF/USCxD5z7gTmAn8Dz2RWxV1WXefm8s9m6i7z8cy+45CKuBucePtxULUxkPHAxsxzJU7gXu8u4uxeIRdwLPAv2w0dTjsC9WsVi/Ddgobj/fdhfwDPalHuHLX/DXw8AcVd3rj1p/5X0Y5v1bDRyC/RimYT/iv2Cj0LO8j6OxEd0/FY7I+rlM1o8q+Hu8H3u99/V5Pyf9AMGsnU3YD/UJVd3mo8rTgZcDlwKHYtlLW33bxb79QixwfYafw73+ufv6eejtny/JsOrv7Rzpn3mv75+0t9i/o+f93PX19p7z5Yr9Tvb593u6n6cXfFlvb3uzf2cLgYF+zOOw9M+b/bxu9nWz/NwPxIqZLE5E0H2Yl/rrXOBXfrwhwIVY5aovicgczH+4EvNx/qP3/aOqehOluRj4bJn1AKjqfhFZi32/Sytt7339zwzbpVmm5Shp6Tr3AG8EvuXvZwKL/PeeJrqjyF5ApqLo+k1pPF1LWaaR5l64AIsLfoh0S7ehoiuVb75VNCoyBBOe4diFJMCT2Je1FovNuxC76J7ERjHnYgI5ALuLnoZdvH29nZHYhdsHu1CfwSyDJ32/5ILdig0CLPT9jsAukl2Y8G7CnPAnYpb+ZEwAD8ZE+zkslGU+JoKJAIzDrIrdvu1g7CIeBWyhQwz6YxfpdN//Zf7ZHsd+UDu8ra3+fpl/1ouwx9B7/HOt83Yu8POzx/s6BhPttf56yv8uwERphL96AfsLzv947Md8LHYzWeT974/d5K7GBGqpn49hmPAc4397YyE9R/h3MNqPN8iP9aQfr49/r0uwm9k6TPj7YjezY/xzDPBtt/n3Osz7Mghjh2+zHLtBzvPP0d//bva+HIYJ87N+Xh71/p6PVZxKZid42L+nvdj3f5D3e7238xg2Un+9qnYRHBEZhAnN5ZgYr8V8iH18v2NVtYt1KCJHYt/xUFXdV7w+ZfubgW+o6o0VtuuLGS+TCmYALrXtBcCnVPW8Ssf37T8DHKSq/1Ri/TDs+jhFVZeLyD8AU1T1AyLyBuBtqvq6gu3/AlzpPuFKx/5nYJeqfqXMNscA16hqxQpibljsAY5ISl+KyH8CDwLfx343Q1R1t6cg7wEGNqqsIzTI0nXLbJx/4EOAnSX8LaV4CLiycIFbz6OxC/tpVd1fQxdvTVvoxxCvbF8TInIUZhk+qaqV4v7uo8NqKG5nMPaE0A8TtiW1Zsv4NCnHYz+4x0t8N/PI9ujajtwEfBdefDLYr6p7kpX+uzwSuzmMxC7ysgHxfsH+1F+dEJEfY5bsh1J2vQi4LYvgOlldAScDyysJbs42E4ZRpsqWqm4UkW8C3xORi7Hf58999QLSLd2ssa8bqDyjRCd/bjlUVUVknfchsYzPB76kqgdEZAX2hPQENoC4pZGCCw0S3QR/fNtRccNsbR2ggfnQBceoV1ubKSHuOdvZBfx37T3q1OazmEXd41HVZ1KWKWYpg1natfItYKGIfFZVt8KLwn4RNqL/yRxtZRXIs4DbMra5ChghIgMy3rCHYy64cnwbeDsWtXAGHenCS7BCNgMLbnSjSQ8ZS2MjBf7iEmT15yYkft0l/hR+MHaeweszYKLbcNcCxEBaENSMuxX+CHysYPHbge8B/1tV/5CjuRVYPddKnEJ6XGpa//ZjIpU1HK3SQFrS5nswn/IdqrqpYPkyPILBn6oGYO60LGzGkh7KkdnSdVbSUWntWGBewaDmekxsoQmDaBCiGwT14ovAh0RkooiMBv4vcJmqFhe2qUSnOrNlOBkbK8jKA5hQJ0VyylFpIA2w+F1skPONRasKB9OSySizDh5twXzs5cgrugsw6xhsjKWw7kWh6DY8XAxCdIOgLqjqSiwV9XrgT8B3VfWRKpqqmMjgKbD9KF1ZLI0HgFNFZDKwRUTeXGbbipZugqre71lvhXQR3Rz93EKZqmA+2DWVfKJb2J9KottwS7ehPt0geInxb9jj8eHAv1fZxnpgqIj0KTOgczI2cWKe0KNbgG9ig0W/Br4rIlcXC6ZbwQOx0MBqmU9HfYY8/lwwN0S5UoxTMMt5V87+FFq6vytYt54OH/Jw8ol5VYSlGwR1Qo1fq+pVOQWxsI39mHCXqzZWOPli1nZXY4OnI4EPYJEwF6RsOgzYWG3/nULLsmRiRAl2Y+OQpWbRmImF9+VhJXCEiBxGuBeCIEihkl83rz834e+B17p1+3vSC9cMI4M/twJLscy6QZgroNSsFl1wsS/nYqhUyjKtzQPYoOObsPDBzQWr12M3Ikiv+Vt3QnSDoP0o6df1bMyqRFdVV6tqkun2COm1azMNolU4TpIcMx2zeOfnbGIrpQfTzqS66dFvAT5CZysXzAof4b7ictMe1Y0Q3SBoP1ZROmxsFrAmCdGqgQXAMSnzh40n3wBdKR4ETiV/pAGUsHRFZAKWOFFN/PsNmKj+vnChJ71s8L4eyJhsUhMxkBYE7cdSSsfUXoTV6agJVd0sIs/TNTZ1AhYrXCv3YoNp21U176BcqcG0y4Crq8lGVdWHReTQJBW4iAVY3Y0n87ZbDWHpBkH7sYzS06ZfhFlt9SCtOM1EOrK1amEOlql2RxX7lorVvRCoWL+hFCUEF0x0X4/VI2k4IbpB0H4sJaX+gIgcgYnk3XU6zhPYwFQh9RLdhdgsE++rYt8u7gWPZphN9tTnPCzELPxcESHVEu6FIGg/VgPDU2olnIdNUVOv6cEfoiBszP27dXEveMTAV6vcfStdU4FPBBam1dKoA1djft20CUPrTli6QdBm+Oj/Krq6GP4Gy3arFw8BJxW8PwqbQLIRwpaHtIG0aeQrcpMZVd2uqn+uMTY5MyG6QdCePE7Bo7+IHI3VAf5VHY+xEIunTSZnrJdrocn/EAEAABKQSURBVFbSBtKm0SSfa6MJ0Q2C9uTFSSpdFG8G/iln+mtZ3KK+jo6U3XYR3bSBtBDdIAgayiN0TLHzWeAvqvofDTjOlcAH/f96hYvVSpp7YTo9RHRjIC0I2pOHgRN95ot30TW0q17cBxzmiQcTyVijt8F0ci94tthY2sMKr5mwdIOgDfF52p4EvgM8kDJrbr2OcwBzXZxH+7gXtmIFapJsudFYEZ7iEpLdkhDdIGhffge8jRLz59WRm4BLsDn9Wv4I7/PJ7cOm1QFLTW4Ht0ddCNENgvblKuDESjMD14FrgVcBqzNMotosCgfTxlOfehBtQYhuELQpqrpbVYurYjXiOFuwIjLXNPpYOSgcTBtPDxLdGEgLggDgrcDOVneigI1YnduHMNG9s6W9qSNh6QZBgKpuabOBquV0lLc8mvYY4KsLIbpBELQjy4GJHsFQTSH0tiVENwiCdmQ5FsI2EthXNMVOtyZENwiCdiRxLxRPJNntCdENgqAdSSzd0+hhoitNqmYWBEGQCxG5Dpsp40RVzTUDcDsTohsEQVsiIuOBS1T131rclboSohsEQdBEwqcbBEHQREJ0gyAImkiIbhAEQRMJ0Q2CIGgiIbpBEARNJEQ3CIKgiYToBkEQNJEQ3SAIgiYSohsEQdBEQnSDIAiaSIhuEARBEwnRDYIgaCIhukEQBE0kRDcIgqCJhOgGQRA0kRDdIAiCJtKn1R0Igp6OiIwG/gHYhc1ue7WqXt/aXgWtIizdIGggInIEMAd4ATgKeA74jYgc0tKOBS0jLN0gaCxXANeq6qeSBSJyPHAScFvLetVEROQgYLfG3GBAWLpB0GheAfy+aNlc4OQW9KXpiEg/zK1yaav70i6E6AZBgxCRXsBM4JGiVXOBU5rfo5bwNv87CkBEzhaRoS3sT8sJ0Q2CxjEZ2KKq24qWt8TSFZELROQ1TT7sDGAdMEFEDgf+CLy9yX1oK0J0g6BxzAQeTVm+FDi0BRbfx4F3N/mYw4H7gAmY2D7PS8fKTyVENwgax2RgSfFCVT0APEgTrV0ROQw4AzhVRKRObQ4WkVtFZESZzYZh0RsTMKv358Cp9Th+dyVENwgax0RgeYl1c4FZ9TyYiPQTo7+/v0hErnORfSVwFyDA2Dod8jPA2d52KQot3enAtcAhIjKsTn3odoToBkHjKCe68zARqgtuba7D3BmrROQU4FPA6cCbMav6fn/VbGn6IOE7gJ9gERqlGA4sAnZiN5mF2A3nJWvthugGQeOYCCwrse5JYGodj/UF4HfAL4Av+98pwIeAd2GC9yB1El1MxLcDP6aE6LrFPRjYCvwXsBfYUMc+dEsiOSIIGoCI9MVSfleX2GQxcLSISJ2SBi4ALlLVRS52n8Ys3WuA7/s278ZiZr9Uh+O9FRP5J4DpJT7HUGCTqh4QkR8AB6mqisj9WFr0S5IQ3SBoDGOBdaq6P22lqj4jIjux+NWnajmQiIwDBmDWM6q6T0SmqOqzvv6TQG9VXSciu4DjRaRvqb5lON4A4C3ALFXdISLPA0cAW4o2HQZs9D4tBj7oyx8Hjq3m2D2BcC8EQWOYRGl/bsJizAVQK+cCdxZamong+v//T1Wv8v+fAVYAx9VwvBOAVaq60t+vAsalbDcccycUsx4Y4k8DLzlCdIOgMZTz5yasAUbX4VjvBH6dY/v7gNk1HG8SblU7pUT3RUu3EFV9HtiMiXJTEJFLfPAvy7aHeCJHQwjRDYLGUC5yIWEDJkxVIyJnYfHA1+bYrdaBrEl0vqGsIj0MrZSlC+ZSqccNpyIicjZ2frK6NH4C3CkiBzeiPyG6QdAYsopu1daeiPTBogIuz+mffRBzEVTLZDqL7mpyWLpO00QXy8TbDpxYaUMRmQ68HLgJ+5x1J0Q3CBpDFp9uTaKLJSWsU9Wbcu63ChhTw3GLLd2lpIe/VbJ0R9XQh0x4YsgrsNC2iqKLhcLdpqr/qKrFhYrqQohuENQZL2c4BUsEKMdGahPdy4A/VLHfNqBvDY/Pxf7qucDJKenF5SzdtTTH0p2Mhcn9D9lEdyo2wNkwQnSDoP7MAJYXRhCUoGqfrg8KvR64Ou++HuVQ1eO9iPQGhtBZTNdihWyKXQzlLN311GEgTUSOcpdAKWZjPuwngGMz1J2YSudBwroTohsE9edkzPqrRC3uhdnAVlWtViCq9akOAbar6gvJAhfxB+g6ODec0pbuNqCmCAEX0LnAQ0m9iaL1vYCPYUkcTwOKTZlUjimEpRsEjUVEemUNJ8rIbEyEKrEFK/7Sr4pjVOtaSKg2XG0osCll+a3AhckbERmIJWxsL9FOzaILTPO/i7Dpj4q52P/+zm8MCylT78J/A5MJ0Q2CxiEil2OPumtE5C8icq+InFNDe72AVwE3VNrWSzxuxoQszzEEE93croUCngLGiMjAnJ/3KKzPxfwJeI27H8D9uWVSnOshuudiYn83cFrK+ncDVxX0YREdQp3GGOzpYVeN/SpLiG7wkkVEPg58DrPQzgO+B/wL8CsReVWVzZ4CbFbVFRm3r2YwbSY2u/DjOfcrZCUWhfAD4JYc+6Vauqq6ChPjGb5oGKX9uVAf0b0IC+26G6umhogM8b8DsHoUvynYfiFwTJn2Gj6IBg2qveB34iNVNe2OmGzTGzgaOwmjsDtoX6w03UPYQMSBnMcdgf2QpmKB0AO9/TVYLdFHgCdUdU+J/ftiN6Le3pd+BX/3YT+iw7EYxx3APFXdW6Y/vYDjgTOBg7BHrX1YWb9VWAWm/cDqQh9ZmfbGAP2xH+xgYLf35zlshHYL9p2O9nY3qepzKe0IVoxlmL8GeZ8Wl7JMfEbXwf72WX/1wQSjt/dpn/evH3AwcCQWE0qe71JEpmDC8iz2uHcwZo1uBZ7BQpSeyvv7KGh/MPAV4HzgbFVNitIs9PUbgN+KyLGqujVn86dh1ldWqhlMOwe4scZCOY8Cn8QHv0Skd5bfIKUtXTDBmuhtl/Pngv1ejqi24I9PYf9KrPDOYOD7/ruZLyLnYdfGUlXdWbDbPVg5ylI0fBANGlfwZgiwREQUK+f2HCYCO4HDMGFLRkDnY6K4GdiD1f78JjadyRJMqNZi+eJHYSKzBvux9MHE5lBs8GIgdtKWY1bAHszvNR6rmv9+YLKI3AvciFkLQzBL51Cs0DK+fH9Bv/djwtTbX494PyaLyGI/5i5MdA7xNkdhP7ytmCWxAxjhfXy/r9+JidRBIrLC/x/k2/THfjgHMMEehInQM37sXXSUzevr/d/vx9/o/TxcRNb75zlQ8Drc16/HrJbdfv4OcUHe5n3r65+tN3bj2IsNRgzCimH3wgYokoInff2c7fNzvxt7ZBsgIvv83D6DCc0u3/aA7zuKjhvec9hsA4OxmRd2+PEP9885CcvdX4GFLi3xfm7HbrKjMeEf5m2r7zfSz9cQ7PH/FaraxeeoqneJyPXAh4EvFq+vwHT8RpORagbTzsJKN9bCPOw8PuR/h5Duqy1mKKVFdyV2rUEFS1dV93qhnOR3DZjhkzHR40LgHlXdAezw4kFfwz7X17G43LQJQUeLyEhVXZfSZsMH0aBBoquqT4vIEdhF0s9f/TGLZQcmDtv9hKXijwmTMZEejQniCiz8YwyW1rePDovv48CKMnfNH3q7h2GWwvmYGO3ASuBtxSy9VMvVxWgosC2xHn2wYAb2oz2IjoGDbVhB6adVdU2ZU5W0PQoT5Of8s+z2/wdiIpRYlnvKWKKJ9bq5oH8Hebu9Cl69vf3lhW35/odhIngYJt5glktf4G8Kvy9ve3+aJV3Ur0MwAe5Ph/gNx85X8mSxC7tgk5vDDs/PL9fuIO/bZOxiOcXbXoTVFngOu/kkk0Luwm7WQ7CEgn3FbRbxdeBuF4ah2E3mcVX9UYX9pmNT0mQll+j609MZwHtyHKMLLnoLgDuwa2gY2UT3KCz8Ko0VdGRxVbJ0ocPFkFRD+zDwVRGZ4e6K5PfTJ+WJ4zTg9oL3dwOXYNbvHOwm3WkwU1VfEJGbMcH+SUp/pgJ/qdDn2lHVeMUrXikv4DWYxfSPwEexp7K/L7O9YDfvoTmO8VHguzm2n4oZF/X4fB/BipvfCpyXcZ8/AG8sse6vgGv9/+8BH67Q1jxgRsG52wL8J/Djgm1+CvwpZd97gbMK3g8Dhvn/t2IG2Qkp+70L+E3K8t6YwZT5u6v2FfV0g6AEqnotBYVkROR/gPtE5I+qmlYDN4lCKDmWkcIG0kfeS3ES+dwXJVHVKwFEZCPZ/cpHUdoiXkGHi244nS3RNAoH06ZgrqfPAXP9yesI4LWAisg47bB++2GlKV88D6paaFX/ArhZ09N4bwC+JSJ9tPPT1PHAWlXNYu3XREQvBEFGVHUpFmj/thKbTMQGb/IMDOX16c7C/LD1JI/olvPprgAmuAtkBOWjF6Cz6L4CmKM2qLkHE+FzsAHwazDXQcJUYI2WCO1S1R+r6ldLrFuHfd4ZRavOpPJNoi6E6AZBPn4DvKnEunFYVEoe8opu3SzdAjaSPVa4pKWrFimwFTsP06gcCZAMaoJ9rsQHeyfmmz0Ru8HcgYliwtHUNuD1MGbZFnIWIbpB0JbcBYwTkTShrEZ0M8fpepjliZho1JNNZLB0PaTyUEwsS7EQs1Cfz/CovhkLKwSLekiqsiWiewL2We8AziyomzAZCxuslkcoEF0/r2f4cRpOiG4Q5EAtlnUO6TPgViO6O4B+Ho1RiSlY7HXe2OFKPI35TysxBNii5eOjFwBvxAbJKrGJDgt7LB3n7k7Msj0JeFhtWqDnsCghMEt3SYb2S/EonesJz8QiWhruz4UQ3SCohntJH/zKLbru/82aIHES9ffnglmuWUS3nD83YQGWCTY/Q3uFojuOjpmTl2Bhpo9hoZdgroeT/f9aLd2Hsck5k5oXs7EbaVMI0Q2C/NyJ+QCLqcbShex+3VnU358L2UW3XORCwp+Bq7AZLSqxCRgqIodiWrQdXrwRvR94R8GgZKHoTqMGn64/KSwtaC/xHTeFEN0gyM8czK87NvEzur9zApVni0gjq+g2YhAN6mjpqupTqvohVc1iOW7ChHwsNrtw4WzG12jnrLEHgFNFZDSW1FUx6agCt2IFc6BxTxCphOgGQU48vvM64LvANhH5KZZ+vFo75/pnpeJgmg/2JANL9WYbllZdqcB3uboL1ZC4F8ZRWUTvx2JzLwDuzxmWl8YfgL8VkSOxELRSWXZ1J0Q3CKrjs5jv8XXA2cB7yVa4PI0slu40YL2m1IqoFbXU9/1YanY5RmD1OupFUtZyChUGxlR1N1aw5gpMgGvCLfG5WPrwdVqiCFYjCNENgipQ1TWq+glVvR34FvBBqn/0zzKQ9gasjGGjyOJiGIUVn6oLLvb7sIGsSvPJAfwei2L4bZ268B6s5kfeokY1EaIbBLVzJRabWm3lr7KWrk+1/h7g36tsPwtbqCy6I+mIJqgXT2G+1Yqiq6o/VNUZWv0URcXtbVfVi1Q1S3hb3QjRDYIaUeO2GuJnK/l0L8EGmmopWl6Jplu6zp/8uFks3R5BiG4QtJ5KPt2/A/6jwX3IIrqNsHR/QUdd55cEIbpB0Ho2AsPSogd82Wwa688FE71yLo7BWMJCXQfyVHU+MLkO0QjdhhDdIGgxqvosHbN+FJMIYaWKXbWymPKTNo7ESh/WXRw9MuElQ4huELQHpfy6M7B5/RptCZadnhwbKGykT/klQ4huELQHpfy6M2iO2HUSXRGZIyLf9v97YZNYfqsJ/ejxhOgGQXuwHpsLsJjJNGGyRCx06xAROUxEJmEpzX/ncx2eAuxV1Xua0I8eT4huELQH95JeRGck9Q/T6oKXa7we+DxwGTZN0WNY2cNLsbTZoA7EHGlB0B78Bfi4iEiR/7YRYVqleB9WQW00VtT7fVix7wupcfbhoIOwdIOgPXgSC8f6WNHyUTRJdD2543TgNFV9Aiv2fTrm4ni0GX14KSAvofC4IGhrRGQsNuPCKFXd6em/u4FBRTPXNqs/E7GbwYOq+vJmH7+nEpZuELQJPhPu3cCrfdFQbHqcpguu92c5VsKy5qpeQQfh0w2C9uJq4K+AX9Ncf24p3tni4/c4wtINgvbiFuAsT/8dTYtFV1V3qOqOVvahpxGiGwTtxUqsZuwUbPbbZS3tTVB3QnSDoI3wcLE7sCnIa51qPGhDQnSDoP24HUuUqHWq8aANCdENgvbjdkx0w9LtgUScbhC0GT6ItgybmnxAq0LGgsYQIWNB0GaoqorIK4HpIbg9j7B0gyAImkj4dIMgCJpIiG4QBEETCdENgiBoIiG6QRAETSRENwiCoImE6AZBEDSREN0gCIImEqIbBEHQRP4/4ThvF+BskkMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 3 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fbc9a2da390>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29eZxkdX33+/7W3vs+aw/TMzDAsDlgMwIa3EBQEyBGExKTYKIheRLzxCQa8eo1xqsJMTdqFmLkEiPRKBh9oqOBIJv4eJVl0AGGdTaYmZ6l97X2qt/zxzmn+nR17VvPdH3fr1e/uuqcU1W/Pt19Pue7izEGRVEUpXnxrPQCFEVRlJVFhUBRFKXJUSFQFEVpclQIFEVRmhwVAkVRlCbHt9ILqIT+/n4zNDS00stQFEU5rXjyySfHjTED2dtPSyEYGhpi9+7dK70MRVGU0woReSXXdnUNKYqiNDkqBIqiKE2OCoGiKEqTo0KgKIrS5NRECETkWhF5UUT2i8gtOfZ/TkT22F8vici0a1/KtW9XLdajKIqilE7VWUMi4gVuA64GjgJPiMguY8xzzjHGmD92Hf+HwMWut4gYY3ZUuw5FURSlMmphEewE9htjDhpj4sBdwPUFjv9V4Os1+FxFURSlBtRCCDYCR1zPj9rbliEim4EtwEOuzSER2S0ij4rIDfk+RERuto/bPTY2VoNlK4oSTaSYWohz9xOHSae1JX2z0uiCshuBbxpjUq5tm40xIyKyFXhIRJ4xxhzIfqEx5nbgdoDh4WH9i1WUKvnHh/bx2ftfwrn+b+hu4ee2LSs6VZqAWlgEI8Am1/NBe1subiTLLWSMGbG/HwR+wNL4gaIodWB8PsY/PLQftxHw3aeOrdyClBWlFkLwBLBNRLaISADrYr8s+0dEzgV6gJ+4tvWISNB+3A+8Fngu+7WKotSWXXuOEUum+cbvXs5lW3vZ0t/G/943vtLLUlaIqoXAGJME3g/cBzwPfMMY86yIfFJErnMdeiNwl1k6G3M7sFtEngIeBm51ZxspilIfnjo6zbrOEDu39HLXzZdz9XlrmQrHV3pZygpRkxiBMeYe4J6sbR/Pev6JHK/7MXBhLdagKErpPDMywwUbuzLPu1r8RBNpYskUQZ93BVemrARaWawoTcZ8LMmh8QUudAlBZ8i6J5yJJFZqWcoKokKgKE3GobEFjIFz1nVktnW2+AGYjSRXalnKCqJCoChNxmzUuuvvbvVntnXZQqAWQXOiQqAoTcZc1Lrr7wgthgi7MhZBcwjByHSEP/vmU8ST6ZVeyimBCoGiNBlztkXQEVy0CDqbzCL447v38I3dR3n66HTxg5sAFQJFaTLmYwUsgmhzCMHhiTAAAZ9eAkGFQFGaDsc11J5DCGbCzSEEJ2ajAETiqSJHNgcqBIrSZMxFE4T8HvzexX9/v9dDa8DbNK4hh0hChQBUCBSl6ZiPJekI+Zdt7wz5m0II3FZPNGEFi5u986oKgaI0GbPR5JL4gENniy/jNlrNTEcWW2lEEykeeuEkW/+ve9g/OreCq1pZVAgUpcmYjybpCC4Xgo6Qn7nY6rcI3CmjkUSKrz9ujVN54YQKgaIoTcJcNJHTNdQe9DG/AhbB3pGZhubzx1OLnxVNpJiYjwHg80jD1nCqoUKgKE3GXB7XUEeo8a6hVyYW+Pl/+BF/ec/zDfvMbItgYsFyFc02gVssHyoEitJkzMeStOdxDTX6YnhixkrjfGZkpmGf6RaCaDzFxLwtBE0QKM+HCoGiNBnz0eSSGgKHjpCP+QbHCBwLJOBt3KXI7RqajiQyBXYqBIqiNAXGGMKJFG2BHEIQ9BFNpEmkGuevH7P9842s8HVbBCftwjJQ15CiKE1CImVIpQ0tgeXDZxwroZFxgvG5lRUCxy0EahEoitIkOJW0If9yIXAyiRqZOeRYBNEaVvhG4ikOjM3n3e92DU0uuISgSfos5UKFQFGaCOeC25rDInAyiRp5QRyzLYJaumXee+cTvPlvHyGVp1o4ZlsEnSEf47YQQXMP5amJEIjItSLyoojsF5Fbcux/j4iMicge++t9rn03icg+++umWqxHUZTchO0may25LILgCriG7AtxLd0yPz4wASy2287GcQ11tS5mSQ10BJvaIqh6eL2IeIHbgKuBo8ATIrLLGPNc1qF3G2Pen/XaXuDPgWHAAE/ar52qdl2KoizH6bZZ0DUUa6QQ1C91cyaSoLs1sGx7PGMR+IEIABu6QozOxZYd2yzUwiLYCew3xhw0xsSBu4DrS3ztNcD9xphJ++J/P3BtDdakKEoOnBhB4WBxfe+ML/jz+/jYt58BFgfhzEYTGFN947cx18U8XwM9J0bgtN4GWNcV0mBxlWwEjrieH7W3ZfNLIvK0iHxTRDaV+VpFUWqAEyPI6RpqQNZQLJliPpbkq48exhjDbCSBR6xsplq0hN53crFfUD4hSCSXC8H6rhYW4imSDUydPZVoVLD4u8CQMeYirLv+O8t9AxG5WUR2i8jusbGxmi9QUZqBSIEYgRNADtdwWIsxhmQqTTyZxhjDsenFvP1oIk0ybdjY0wLUJlh7cs5VF5Dn/eKpNCLQ5qquXtcVAhobHzmVqIUQjACbXM8H7W0ZjDETxhjHZrsDeHWpr3W9x+3GmGFjzPDAwEANlq0ozceia2j5v37IZwlBJF67i+EXf3iQsz56Lxd/8vt8+r+eZ2Qqktl3aHwBgMHuVqA22UoluYaSaQJejx0jAI9Af3uwZms4HamFEDwBbBORLSISAG4EdrkPEJH1rqfXAU6HqfuAt4hIj4j0AG+xtymKUgcK1RF4PEJrwFtTi+CLjxwAYCGe4o4fHeKFE7OZfbtfmQRgMGMRNEYIYsk0AZ+HdV3WxT9tXDObmzSFtOqsIWNMUkTej3UB9wJfMsY8KyKfBHYbY3YB/1NErgOSwCTwHvu1kyLy/2CJCcAnjTGT1a5JUZTcFIoRgOUeWqihELQGfEy5JoLd9cRiSPDZEUsUHNdQLaajjc3F2NTbwsmZWMFgcdDnYV1XS2Zb5wrUUJxKVC0EAMaYe4B7srZ93PX4I8BH8rz2S8CXarEORVEKk4kR5MgacrbXyjU0E0kwMh1h+/pOfnl4kL/47nPsH51nTUeQ0bkY++3q38GeGrqG5mOs6QgRiaeLuobW23EBsGoKnDU3I1pZrChNRMY15MstBG0BX81cQy/ZGTx/du05/NZrt2Tuus9Z10FPqz/TBmKwhsHisbkYA+1Bulp8eV1Nccc11LkoBE68oFlTSFUIFKWJiCRSBH0ePHmmcbXUMEbgNJRb22FdcDd0Wxf8s9d2sLYzxLTtMtrYXdsYwUBHkK4Wf2GLwOdhrVsInBhBk7qGVAgUpYmIxlN53UKAHSyujWto2r4Q97QtHYt5ztoOBjqCmee9bQFaA96q3TLxZJqpcIL+9iBtQR8LeX6OeMoSAnfH07aAF49osFhRlCYgkkjlDRSDFdx1t2auBueOv7vFavPgXJi3rW1njW0leO1Mpc6Qv+q7cWfa2fquECG/d0kGkZt4Mo3fHoTzmi29nLuuAxGhs6X6NZyuqBAoShMRjhcTAm9NKnwBpsNxgj5PxgL53SvP5GPf3su2tR2s6bQsgraAFxGhq8Vf9d34sRmrRmF9tyUEsWTuKmEnWAxw9+9entneGfI3bYxAhUBRmohoIpWzhsChNeBlIVYrIUjQ3broFvr1yzbz65dtBmCt7RpyKno7W3xVu4aO20KwobuFkM+Td8ZBPJXOtNNwU4s1nK6oEChKExFJFIsR+GqWPjoVjtOTo/snwHU7NrIQT/HLw1Zjgc6QnxOusZGV4LSv2NDVQsjvzS8ESauOIJtWv69m1tDphgaLFaWJiJTgGgonUjXpBDodSSxp7Oamty3AH7zxrEzQuBb++eMzEbpb/bQEvIT8HqKJPK4hO1icTdDvyetOWu2oEChKExFJpAu6hloCXowh70W0HKYLWATZ1CJGcHw6ynq7Wjjk9xJN5hY0d4zATdDnrcnPfTqiQqAoTUS0iGuoLWB5i2uRQpodIyhEZ8jHbDRBOs94yVIYnYux1g5Ch/yWoMVztJV26giysSwCdQ0pirLKicRTtBaxCKD6VtTGGFsISrMIOlv8GAPzVQhQOJ7MCJkTA8h1h5/XNeTzEFOLQFGU1U7xYHFthGA2miSeStPXVroQQHXVxdFEOvOzOe6vWI7gr+UaWn4OCqWcrnZUCBSliYgUSR8N2j2I4lVeEN2pnKWw2OunOovACYQ7P2OuLKC8riGfuoYURVnlpNKGeDJdMGvIuUDGU9VdEI9NO0IQKnKkRWeL5dKpJo8/kkhlLBrnZ8x2DRljCriG1CJQFGWVEy0wnczByaap9oI4Yuf0byzXIqgwhTSdNkRdGVEhvxMjWCpoTvA4Vx1B0OfJjNRsNlQIFKVJiBQZSgMui6BKITg2HcHvlcwIyGJ0VRkjiCaXzlkIZSyCLCGwf66c6aP+2ohgqXz+gZe4/K8eLPn4kelIVVlVhVAhUJQmwRlKUzhGUDshWNcVytvuOpvFNtCVxQicn601kGURZP0ciZR1IfV7l6/LmdHQqMyhzz+wj+Mz0ZIskOMzEV5760N8/sF9dVmLCoGiNAmLg+tLiRFUdzE8MhlmQ1dpbiGAjqAPkcpjBOEskXOC3nktghyDeRYtgsYGjEsZDXpgdAGArz9+uC5rUCFQlCYhM6aykGvIW71FEEum2Htslgs3dpX8Go9H6AjmnypWjOxZzEVdQ3mCxdAY11DCJbRzJcRFDo5b09zG5mJ1iWGoEChKk9CoGMHTR2eIJ9Ps3NJb1uuq6TcUzuMaynbzONlQ+dJHoTEWwSsT4czj+RLcYQfHFjKPD0+GCxxZGTURAhG5VkReFJH9InJLjv1/IiLPicjTIvKgiGx27UuJyB77a1ct1qMoynIy84rr7Bp6/NAkAJcOlScEHaHK+w1li1zGIsi6qMcKBYsLVCPXmpfHFy/spcRFDo4v0NsW4FM3XJDJsKolVbehFhEvcBtwNXAUeEJEdhljnnMd9jNg2BgTFpH/AXwG+BV7X8QYs6PadSiKUphoKa6hGlgEhyfCrOkI0lNiVbFDqIpeP9nxj0xBWTy3ayhX+mimGrkBrqGp8OIUuFJcQ4cnFrjizL7MPIdaUwuLYCew3xhz0BgTB+4CrncfYIx52Bjj2DOPAoM1+FxFUcqgJNdQDeoIxuZjS2YSl4pV2VvZ52biH44Q5Lm7LxwjcNxJ9XcNua2A+Vhxi2Aumszb0rsW1EIINgJHXM+P2tvy8V7gXtfzkIjsFpFHReSGfC8SkZvt43aPjY1Vt2JFaUJKyhqqQbB4bK4yIQj5vRVfhLMD4T6vB79XlrWYcFxeubuPNs4icAfF50pwDYXji1XT9aChE8pE5NeBYeD1rs2bjTEjIrIVeEhEnjHGHMh+rTHmduB2gOHh4eYr/VOUKimljsDjEfxeqSpGMDYX49x1HWW/rhqLIJxD5DpD/mVul4IFZQ0MFs9GE4iAMcVdQ+m0sdtn1O9yXQuLYATY5Ho+aG9bgohcBXwUuM4YE3O2G2NG7O8HgR8AF9dgTYqiZFFK+ihYF8lKLYJ02jA+H8sMpy+Hanr95Ip/WFlIS++2S3INNcQiSLK+M4RI8awhx6qpp0VQCyF4AtgmIltEJADcCCzJ/hGRi4EvYonAqGt7j4gE7cf9wGsBd5BZUZQaMRdLEvJ7cl4E3QR8lQvBdCRBMm0YKLG1hBtrvGSVwWK3EISW1yUUcg0ttq6uvxDMRBJ0tQZoD/qKZg1lp8bWg6ptDWNMUkTeD9wHeIEvGWOeFZFPAruNMbuAvwHagf8QEYDDxpjrgO3AF0UkjSVKt2ZlGymKUiNmIwk6Skg9rEYIRuesZnMDHaV1HXVTjUUQjqcIeD34XC6fjtDyuoRTyTXUGfLREfQVjRE40+Lq6RqqyTsbY+4B7sna9nHX46vyvO7HwIW1WIOiKIWZiybpDBX/lw/4PBXHCMbmLK9vpVlDlVoE0URqWUpoZ4uPE7PRJdsKdh9tcLB4U28rHSE/87HCMYJGWARaWawoTcJstESLoIoYQTVCUM2EsESOGQOdIf9y15D9/v6CBWX1twgsUfbTHvIVTR8Nx4tne1WLCoGiNAmz0WSmy2chAlW4aEZtIVhToUWQShuSFVgjOYUgR8uKQsFin90pNZ6qf1LibCRBZ4uP1oC36FhQxzXUFjy1s4YURTkNmIsk6GiAa6g14K3oohXM0zq6FOLJ9LK7/M6Qj2givcTnX0gIRKzU2UqEqBxSacNczLIIWgNewrFiQlBatlc1qBAoSpMwa7sjihH0eohXGDCttJgMCg+cL0YiZZbNGHCsH3cwNp5KI7J495+Nz+MhWYPhLzORRN4Gek6r7c4WP20BHwvxIumjGiNQFKVWzEYTmdnAhag2a6gStxBUl8dvzSFeeqHMjL90xQniyTQBrwc7e3EZPq8saRFdCU+8PMmr/uL7/N5Xnsy5/5DdUnqor5XWYHHX0IK6hhRFqQXRRIp4Ml2SRVCta6hSiyDfMJlSsC7w2RaBdeF05+nHkrkH1zv4vR6SVcYIHnnRaoHzzNGZnPv3j1pCcNaadssiKBIszu6jVA9UCBSlCXDcIyWlj1aZNVRJMRm4ZghU8NmJVK4YgSV67qln8VQ6Z+qog88jJNPVWQSTdmfRPEYH+0fnCfo8DPa00hrwEUumC8YlFuwYQqvGCBRFqQbHX13PgrJoIsVsNMmazvKLyaC6CWG5soacNtiTC5mONhnXUD78Xk9mrnGlTNtCMBtNksoRb9g3Os+ZA+14PUJb0PqZwwWsoHAiScC3tFiu1qgQKEoTkLEI6hgjyNQQVGgRVJPHnytryHFRjc/FlxxXyDXkq0HW0ORC4VkDL48vsHWgDVisFs6em+AmUufOo6BCoChNgTMIpaul+LCYSmMEY/OVF5NBdZW98ZRZJgQdQR8Bn4fx+UWLIJfl4MbnERJVZg1NLSxe/KfDS4XAGMOxmSgbu1sAMhZBoTjBQixVV7cQqBAoSlMwXsbdesBbWTvo0dkqhaAKiyCRw/cvIgy0BzOWChS3CPxeD4kqW0xMhuOZczCTVdk8sRAnnkyzvstynzkWQaHMoUgiSWsdM4ZAhUBRTknG52P88d17ePKVSZKpdNUpjePzlkXQ117cIghW6hqar7yqGGoRLF4ene3vCGbWBXaaaQFfu88rVdURGGOYDsfZ0me5fqazhOD4tNX7aL1jEQSKWwT1HkoDDR5MoyhKafz+v/+Uxw9N8v1nTxD0e7lwYxdf/q1L8+a/F2NiPkaLv7SKX8c1ZIwp6/PGZqOIQG+Zs4odMsHiGsUIAAbaA4xMLzaeK5Y+6vN4qhLd+ViSRMqwpb+Nx1+eXGYRHJuJALChyxICJyW0kEUQjqXqWlUMahEoyinHkckwjx+a5LpXbaC/I8jkQpxHXhrjh/vGK37P8fkY/R2lXaADXg/GUPad8dh8jL62YMXZLe4WE9/YfYRv/2zZfKu85PP997cHl8QI8gmGg9VionKLwIkPDPVbFsGMa0g9wLFpSwjWd1uuIUeYC1UXhxPJuhaTgVoEinLKce/e4wB86JpzGOxpIZEyXHHrg/zH7iO8/uyBit5zfD5Of4nZPM4FtdhFM5tqislgqUXwf397LwA3XFxo/Pki+dba324JaSpt8HqEeLJYHYGnqjoCp4ZgqK8VWB4jOD4TJeDz0GdbTY7Lp1C/oXA8VddiMlCLQFFOOe7de4LzN3SyqbcVESHg83D1eet4+IXRilskj9t366XgFoJyGK1SCJyL4tGpSNmvjeexCLpb/aTSJtPqOd9xDlaLicotAicov6G7hRa/d1nW0Pi8VXDnuNzaMsHiAhaBZg0pSnNxfCbCzw5P89YL1i3Z/pbz17IQT7H75amK3nd8Ps5Aqa4hRwjK9JVPzMfprzA+AFbGTovfy48PlO8CS6RMziCwcyftxB1KKSirxiJwp9B2t/qXWQTz0eSSDrCtdvpooZkE4Xj9XUMqBIpyCnHf3hMAvPXC9Uu2X7KpB4Cnjk6X/Z7GGCYXyrAIvJVZBNPhON2tlQsBQHvIx0snrV4860qsUE6lDan08joCgJDtboq4haBYi4kqLAInVbWvPUBXi5/pSIIDY/M8+cokYF3w210X9aDPS1vAy+RC/ill6hpaxTRiCpJy+nHv3hOcvbadMwfal2zvavUz1Neat5FZISKJFGlDSbMIYNEiKCeNM55MsxBP0d1avIVFITrcF0l/aZcnJ8vH71ue4eRcQKP2QPpiriG/r7qsobG5GN2tfoI+L10tlkXw5r99hF/6wk8wxjCXZREA9LYHlrTBcBNPpkmmzenhGhKRa0XkRRHZLyK35NgfFJG77f2PiciQa99H7O0visg1tVhPrUilDS+dnFuSdVANI9MR7n3mOO+7czcXfeL7fOvJozV530qJJlJV56c3K8lUmnQN+ta7GZ2L8vjLk1x7wfqc+y8c7OaZkfKFINO0rET3QrCCGIHTX6enWiFwXSRL/XzHhZXTNeTPYRF4819U/Z7q6gicGABgCYErRnBsJmpZBFn9nnrbgkwsLM0ucsjMIjjVs4ZExAvcBlwNHAWeEJFdxpjnXIe9F5gyxpwlIjcCfw38ioicB9wInA9sAB4QkbONMSt6u/zjA+P8v/e9yN5js3brXh+feeeruOb8tRXlcc9GE9zz9HE++b3nLDPP72Xb2nb+9D+eYv/YPP/zTdvKNv2Msf5YnfWMTEfYd3KOS4d6S/Infv6Bl/j7B/fRFvTR1eJneHMPH/v580rOLHGTThs8eQZ9VMJLJ+e4/7mTdIZ8vPPVm2pqFu8fnSfktzo/loMxhv965jjPHJ3h0PgCD70wSmeLn/dcMURrwMtV29dmUgYr5T9/OoIxcP2ODTn3n7e+k+8+dcyaK1BC8ziHzKjDEs9jJTECp3CqFq4hh1KFwKkEznWnn0lJdYSgaLC4ujbU7syp7lY/Tx9dFIKnjkwzF00scQ0B9LUFODkbJRdOWunpUFC2E9hvjDkIICJ3AdcDbiG4HviE/fibwD+KdQW7HrjLGBMDDonIfvv9flKDdS3ji48cYDaa4BdetYHJ+TijczHOXd/Bues6AeuP5VP/9RxfffQwgz0t/NYVQwx0BLn7iSP83lef5IYdG/jLd1yYKQsvRjpt+NH+cf7w6z9jJpJgeHMPN10xxPBQD31tQf58116+8IMDfO2xw/zLTcMMD/Uueb0xhhdOzPHTw1MkkmmOz0a5b+8JpiMJIvEUAa+HN29fw4WD3fzNfS8QTaTZ2t/Gbe++hO3rO3OuKZ5M85n/foE7fnSIK88eYKA9SCyZ4t69J7hn7wn62wIYYPv6Tj729u1stV0Uo7NRXp4Is21NOz1tAaKJFNFEiu8+fZxPfe85Xn/2AJMLceaiSXxeYXw+xvkbujh3XQe/ftlm/vHh/Tz0/Cjnru/g915/Jpdt7Vt2rj7/wEvseuoYL0+EM9sfeH6UP7pqGxu6WljXFVoigC+emONv7nsRn8fKrHn92QOs6Qzy33tPsHdkBo9HeN1Z/cSTae7dewK/VzgwtkB/e4C//qWLmAon6G2zpkQND/XizRKzaCLFkckws9EkP94/zt/e/xJg3XneuHMTB0YX+Ky97S/veZ63X7SBK7f1c+0F60rq8gnwwolZ+tuDxJNp/u0nrzC8uWeZW8hhS78lXocnwlywsauk94fFQGSpf7fOHXM5FsHUgmMRVCcEHcHF81aqEDnH5YoRuC0CY0wJLSaqG0wzNh9jx6ZuwLII3FXNlhAkl7UC720L8Pzx2ZzvF27AdDKojRBsBI64nh8FXpPvGGNMUkRmgD57+6NZr82ZOCwiNwM3A5xxxhkVLfSVyTBfe+wwtz18ILMt4PWwdaCNg2MLINYf/+/83Bb+9C3nZEbnveeKIf75kQN89v6XeOzQJDu39PLGc9ZwzfnrCPk9vHhyjs6QH59XeOrIDD6v4Pd4uO3h/fzk4ARrOoJ84d2XcPV5a5cU2/zVOy7iFy8e5JZvPc1vf/kJvvuHr2OzXZr+7Z+N8OFvPb3ETxvwejhvQydXnNVPe9DHbCTBd/Yc49t7jnHxGd28+zWb+cx/v8ANt/3/fPAt53DDxRszdyejc1H+6eED3PPMcUbnYvzK8Cb+4vrzMz/jCydm+dT3nqct6KU96OeB50/yS1/4Ma/bNsDPDk9lUvq8HqG3LcBMOJH5B+wI+nj22CzdrX46W3wcngxz6VAvzx6b5aEXRvmnHxzA5xHecM4anj02w423P8qajmAmQ2NLfxseEX58YILXnz3ANeev43eu3Mq9e0/w8e/s5ZGXrEEfl2/t49D4AoM9LbxreJBPfvc5FuIpQn4PQZ+XXU8ds86Tz8POoV7mogn+4aH9eD3CpUM9dLVYvtvnjs/y3jt3L/nbGOpr5bKtfWzsbuHMNe1MLMT5q3ueX1LxedX2tXzwmrPtvxnrYj0xH2M+luRrjx3mq4++wnefOsYXHjnAP737kswNBliibgw88tIYu1+Z5Ef7xulrD/Lwi6NWAZf9+/37X70479+v87fx8sRCWULg/AxOg7NiVJI+OhV2LILqXEOVWQTWzUEu15Dz9x2NpzJpocXrCKqzCByrurs1sKQN9csTC8SS6ZwWwcRCPGcl9+KYylPcNdQojDG3A7cDDA8PV/Sb+stfvJBfvHgjI1MR2oM+5mNJHn5xlBeOz/Ebl29GgDdvX8vlZy69W/V5Pbz/TdsYHurl9h8e5CcHJvjOnmP0tQVY2xniuTxqHvJ7+MQvnMd1OzbmLbvfuaWXO397J7/wjz/i3Xc8xp++5WweeXGMb+85RmvAy84tvXzs7dvZ2N1CT2tgmQvmd19/Jt988gg3X3kmXS1+Xn/2AH/yjT18+p7n+fQ9zzPY00I4nmI6HEdE2DnUy59dey7vfPXgkvc5d10nX33fon4/f3yWP757Dz99ZYqLBrt4zxVDnDnQzpOvTDGxEKMj5CedNqzvbuGmyzfnrSZ9+IVRHjs0yQ0Xb+DcdZ0sxJJ84QcHODYT4dh0BJ/Hw7GZCPFkmg9ctY0/evO2zE8YffcAACAASURBVD/Db1y2mYs3dfPSyTmeOzbL//rZCGs6gjx1dJrdr0zhEfju+1/HhYNdxJNpDk8u8OKJeV6ztTfzzzgXTdAW8C05b4cnwuw5Os2GrhCptGFkOsJ39hzje08fX5LGt3Ool3dfdgadIT/RRIo3nLNmmZuqrz1IX3uQj7xtOx+46mwePTjBh775FNd+/n9z1fY1fOCqs9k7MsOt//0C4Xgqc3Fb2xnkxGyUa85bR09bAK8H3vu6rWwp4F7abBcpveKymEphoVyLIOMaKt1Dm4kRVJE+ClkxghLbXGQsghwXeMciiCZTBWMJDtWMqowmUoTjqcz/ujMz2eHIpHUztSxY3BbIBNuzReJ0cg2NAJtczwftbbmOOSoiPqALmCjxtTXl0qFeLh1afF5q5SLAZVv7uGxrH+m04dGDE/zzDw/yo31j/PLwIGNzMTpb/Pz6ZZtJJNO8dHKOK87q5+y1HUXfd1NvK//22zv57S/v5o/vfsr+rF7+9T07i/rHt/S38aFrzs08H+gI8pX3vobnjs1aIndijq4WH31tQW64eGPBC42b7es7+e8PXLls+xvPXVPS693Hu1/TFvTxwWvOKfn1F2zs4oKNXbzjEvjYz58HWDGXg2MLGGO4cNC6Mw74PJy1poOz1iw937lcNGf0tXJG39IYwTsuGSSVNiRSaV44MUfaGHYMdpcV+2gJeHnjuWu494+u5GuPHeafHznAA8//CIBXberm1Wf0sOOMbt507ppl//Cl0BrwMdAR5OXxhbJeV7ZFUEH6aMYiaKnOInAu3B6BtLESNnw5msm5cdaZPaoSFi2CSDy9eFyx7qNZQjC1ECeaTLHe7g+UD2f4jyMA7nMx1NfKkSlLwJcHi+0BOvPxZX8XjRhTCbURgieAbSKyBesifiPwa1nH7AJuwvL9vxN4yBhjRGQX8DUR+SxWsHgb8HgN1lRXPB7hirP6ueKsfqKJVOaPzc0VZ/WX9Z4XDXbz4J++nicOTXLZmX20BbwVNxgDOG9DJ+dtyB0nON3pDPkzftha4vUIXo+36vce6AjyR1dt4zcv38yDL4wS9Hl4+4XraxJQ39LXxj575m2pOBZBW5kWQTnpo9PhOAGvp+o715QdA2oL+piLJomn0kV7FzkX7lwX+IxFkEiVJAS56gguv/VBook0L9/69oLrmI0sHQfa5RKCCzZ28b2nrdYhy1xDdkfYiYXYshuUjIif6q4h2+f/fuA+wAt8yRjzrIh8EthtjNkF/AvwFTsYPIklFtjHfQMrsJwE/mClM4bKJZcIVEpXi5+rzltbs/dTVpaetsAyF1y1XLqlh9sePsBtD+/n8EQYjwd+/w1nsak3fxZUuQHHytJHE3S3+qu6eQEyKbmtAa8lBMk0xeLPiQLB4lDA2hZxCUGh/kk+rxUjcFxSJ2aimRqEWDKV6YeUi2yLwHEBXb61j7Wu4rjsYLFT6DcxvzyF9HRyDWGMuQe4J2vbx12Po8C78rz208Cna7EORVntvOGcNdz28AH+5r4X6W8PMD4fZ2wuzh03Ded9jXMxKbVNQSXpo1PheNUZQ7DY8dSKZ8RKEqNCF/iA14OI1WLCiXkUdA3ZVlsybfB7hQdfOJnZd3Bsge3rOzHG8I3dR3iLHd9xmLVTaJ3U3osGu/nY27fzruFNfOUnL2eOa88RI4ClIy4dnBGjpRYDVopWFivKacTFm7rZ0BXi115zBrs/djW/eflmfrR/rGClejiWwiOFs2XcVBIjcCyCannjOVY86ee2Wa7VUsQoXsA1JCK0+L1EEqmMq6twsNja57iH3NPNHJfckckIH/7WM3zrp0sLQp2+Ql32XGivR3jfz22lq8W/pL4iuwZk0TW0XAimw3E8svw1tUaFQFFOI3xeDz/8szfy6RsuAKwst2gizU8OTuR9zUI8SVvAV7LbprL00XhNhODKswfY9+m3cskZPSWvwUkLzXeBD/m9RBOLweJCguhMOUvYjedmIgkCPg9ej7Dv5BwAR+2gb3aX1Fn77j07WwiW1lcM9iwNOrcGfIT8npxtJqbCcbpa/DUt2MyFCoGinGb4vJ7MRX14cw8i8PSR/K0nwrFUpstlKVRaR1AL1xBYLp5y3FOFYgRAxiIoNVgMixbBTCTBQHuQdZ0hjtmTzo7aw2WWCUGWa8iNu/VGruB3X542E1PhRNXV2qVw2tQRKIqynLagj6G+tryVqbBoEZSKzyOIlB4jcOb01vKC5dzdO8VihSh2gQ/6PXZfLVPwOHC7hqz3nI0kMtk/MxHrQj0y5QjB0nqO2ahlPeRKIHF6BW3uyx3U720L5IwRzNTI5VYMFQJFOc05b31nwWZ04Xh5FoGIEPCWPsB+IZ4imTZVN5xz4y+jqG2xxURu90mL32uljzrB4iKjKgES6UWLoLPFh0ckM2TGsQRGprMtgmReX/5Za9oZ6mvlL99xYc79+YRgKhxfknFUL9Q1pCinOedt6OTwZJi5aO6e9guxJK3+8u75Aj5PyXUEteoztOTzvaXXMiSKVAyHynINLbUIZmyLoKc1wJRdPT0ybVkCc9HkksEzs7Zo5KI96OMHH3ojV5yZu76ory2QM320VkH4YqgQKMppjuNucHzY2UQS5Q82Cfo8JbuGpmvUZ8hNOXGKYhf4FjtYHCtFCByLwBUj6Grx0+WaNnZiJpoRneMzi1ZBuV1h3RSyCGopsPlQIVCU05w1HZbrYHQutxBY1e/l/auX4xpy7pTrEiMooSV0sWBxyO9d0uep2KhKIDOu0hGC7hY/0+EExhhmo0kGe63MH/dM4vnY8qEzpdLTFrBTXBddYbGk1buo2rYdpaBCoCinOWs7rcrUk7O5ByhFE+myK+ADvtKFwJlFUMsYQTkWgSMW+YSgLeglEk9mLILC3UcXs4ZiyRTRRNquA/CTTBvmY0nmogk22fMsnGZ7YPUFaqmw04DTdmI+utj00Blq011lI79SUCFQlNOcUiyCci9Q5QjBXFZrhVpQTgfUWLJwsLg14GMhnsoM6Ck07csRk3gq7SoQWywIOzkbI5EybMphEVQzWzgjBK7ut04jv1oKbD40a0hRTnNaAl46gj5G81oEuRsjFiJQRozAuYutpKNqoc+HUi2CtN1KIrcQtAW8hGNJ5p2RnQXOhd9VWZypC2jxZ4TU6SCasQgiS4Wg0p5AjktpLuoWgtoH4fOhQqAoq4A1ncH8FkEyXfIgeIdyYgTzsSQitW2M5tzdx0uJESTTea0BsCyAhXiK+WiStoC3YJWuEyxOptJL+vy025PTjk5aQrC2M0TA68lcrMGxvCq7pLbnEILpTOxFLQJFUUpgTUcoZ4wgnbbGM4YKdM3MRXmuoSTtZbSwKIVgGeMyi80hbrdrKMbmY0Ub77ldQ56k9fOEfN7MxdipIegI+axMItt9Y4whHE9WbhHYQuN2DS1mY2mMQFGUEshnETj+8/JdQ15iJbqGFmLJZR01q6Vc11Ch1tLOZLbR2WhR95UTSE6kTKaRX9C/KASZ4TJBHz2t/szFOpZMkzaVD5BZdA0tupo0RqAoSln0tAaWBC4dnItZPdNH52PJmsYHwFXhW0r30aQpKATOZLaxuTIsgmSaVNq2CPyeTJuJw7ZrqCPkp7tlscgsM0ms0qyh0PJg8XQ4TsDnqfg9y0EtAkVZBXS2+JmLJpcMSwermAzKtwiCPg/xZGkzoubrYBH4vB48UrprqFBKqGMRnJyNFnXdBDIWQTqT0x/yewn6vLQGvEvmDruLzJzzXKlryBHS7GBxTw2G/ZSCCoGirAKcO9bsNhMVWwRlZA3NRWtvEYAlBk476EJYweJCMQJrbbmGw2eTCVIn04uuIVsculsWL/ydIf8S11C4ytnCQZ8Hv1eygsW16+haDBUCRVkFdGc6ZGYLgR0jKDdYvMKuIbCmhWXPD85FIpXG7yuQNeS6OBdzDQVcweLMubOtqS7XRbkt6KWrxc90pDauIRGhI+RnPrb4+5sOJ5bMPa4nKgSKsgpwLhjZcYJosjLXUDlZQ/N1sgj8Pk+m+Vsh4nYdQT7cF/+iQuByDUWz3GpO0LY14MXn9dAZ8mcG3mSK1aoYMt8e9C2pLG5UnyFQIVCUVUFXaz6LwMl8qcA1VGob6jrECMDqBFpKHUG8iGvIbRG0F2nH7Q4WL1pTtmuodelQ+k6XO86JEVTqGrLW5lvqGook6Gk7DSwCEekVkftFZJ/9vSfHMTtE5Cci8qyIPC0iv+La92UROSQie+yvHdWsR1GalXyuoVii0vTR0mIE6bRhPp6kox4WgVdKsggSResISrcI/F6XRZBM4fNIZlhNV4t1d76lvw0g03J6NprMuIaqKaprDy0VgplI5d1My6Vai+AW4EFjzDbgQft5NmHgN40x5wPXAp8XkW7X/g8ZY3bYX3uqXI+iNCUZ11Aei6DsXkNeD4mUIZ0ufEceTqQwhvpYBF4hWeTzwcr5L+Qacrtrik1qyw4WuwW0zb7Iv2rQunw5F+nZSGIxWFxFqme3K+YQS1rdUivtZlou1QrB9cCd9uM7gRuyDzDGvGSM2Wc/PgaMAgNVfq6iKC4cN8VsthBUESOA4uMqF/sM1f7O1e8tzSop5hpyWwvFLILMdLaUIZZML8m2OjFrFeydOdAOuM55NEG4yvRRsGYSOEVk85n2FqeHRbDWGHPcfnwCWFvoYBHZCQSAA67Nn7ZdRp8TkWCB194sIrtFZPfY2FiVy1aU1UXI7yXk9+TPGiozRhAsVQjsLJd6WAR+T2nBYitrqPDPF8i4d4pfWAM+TyZYHHRlW11yhuX5fvWQ9d2xCOaiSSJ2sLiaGEFPW4CphTjGmCV9jhpB0U8RkQeAdTl2fdT9xBhjRCSvHSci64GvADcZY5zf7kewBCQA3A58GPhkrtcbY263j2F4eLi4vagoTUZXi39Jf3xw1RFU0GsIihd0zWUsgtpXv/q8paWPFssaAnjog6/n8UOTvHn7mqLv5/cK8WSaWGKpRfCeK4Z424XrWddltf3OxAgiCSJx6zxV4xrqWTLzoLEWQVEhMMZclW+fiJwUkfXGmOP2hX40z3GdwH8BHzXGPOp6b8eaiInIvwIfLGv1iqJk6GsLLpt7m50LXyoBb2lCsGC3dl5p11CgQB0BwGBPK4N26+hSPtexCNznzeORjAjA4kV6NppgIZ4k6PNkAsuV4KSKTi0kMoWBp0uMYBdwk/34JuA72QeISAD4T+DfjDHfzNq33v4uWPGFvVWuR1Galv6OIOPzSzuQRrKqY0vFnU9fiIxrqG5ZQyUWlFVxAc7GSZ2NJlMFz1tbwItHYDaSZDZSffFXrz2JbCocZ7bBrqFqz96twNUisg+4yn6OiAyLyB32Mb8MXAm8J0ea6L+LyDPAM0A/8Kkq16MoTctAe5CxuaVCYPm5PQV78OeiXNdQPS5YPo8nMzu4EMWyhsolYFsixUZ8igidLX5mowlrcH2VQuC0m54Mxxenvp0qrqFCGGMmgDfn2L4beJ/9+KvAV/O8/k3VfL6iKIsMdAQZm49hjMk0Kpur8ALlXFhjRYTA6ZZZr8ricKR447t4sniwuBzcweJid/mdIT+zkQSzkSSdVYqhYxFMh+MNDxZrZbGirBIGOoIkUmZJ5tBsJFnRxaTc9NFiaZmVYPUaKvz5xhjiNXYN+e0+S1aMoPD7Wv2GamMR9DoWwULCFYRXIVAUpQwGOqzs69G5WMalMxtNVJR5UqpraD5mBUkLVfZWSilZQ07BWbkxkEL4vWIPpik+2c1J+ZytQRVwR8iH1yNMzMeYiyYyPY0agQqBoqwS+tutO8pbvvU0r/qL73N4IsxctDKXRbAMIajXXauTvVMIZ32FZhaXixMsjiXTBItkW/W1BRift4K7TjpppXg8whm9rRwaX2AuWpklV/FnN+yTFEWpK2tsi+Cnh6eJJFJ85r4XLJdFJRZBiTOD6zGUxsFfwjwCRyhq7hpKpYmV4BrqawswWSOLAGDbmnZeOjnHXKwyS65SVAgUZZWwqXdpnvyBMevOspI71XJiBPWyCHwlzCNwhKqWrilnFkMkkSpaINbbHiCSSJFMm5rMDjh7bQcvT4Q5NB5mvatmod6oECjKKiHo87Lr/a/lqu1r+aVLBjk6GWY2Ut8YwVw9XUO+4q4hJ6spWGbldCECPg8L8STJtCkaBO9rW5wXUG2wGGDb2nZSacPzx2fZ3FdaAVwtUCFQlFXERYPd3HHTMNvXdzAXSxJLpiuKEZQcLK6jL9vvsYK2hXDmCtc2WOxhasGq0G4r0juor22xPVotXEPnre/MPB7qa6v6/UpFhUBRViHudgoVWQROHUHRyuJkXVJHwZpZXCx9dNEiqG0dgVPZ21rkZ+ttd1sE1Z+Hs9a0Zx6rECiKUhWbelsyj6uKERSwCIwxjM3F6G/P2zS4KqxgcTGLwBaCKpq95fpch2LzC9yuoVJ7GRXCKQQEGOpX15CiKFWw2XU32VFBQ7hS0kenwtaIxg3dLXmPqQYrn7+IRWA31atti4nFi3Frka6qTu3GBRs7M5PLquWLv/Fqzt/QyRm9jbMIGpeoqihKw2gP+hjsaeHoVKSiYSmldB8dmYoAsLFOQuDzeDAGUmmDN0+vpEyMoMx5C4VYMsimiEXQGvBx3weurJkIAFxz/jquOT9X5//6oRaBoqxS7rr5Mm7YsYGLNnUXPzgLj0fweYR4Kn+vn5FpSwgGe+pkEditpQtZBfWIEbhdQ6WI6DnrOupSWd1I1CJQlFXKYE8rn7/x4opf71TY5sMRgrq5hjyLrbDzdQGN1yl91KFRvX5WmtNbxhRFqRtFhWAqQovfS09rfSpgfbavvlBRWT0sAndhWLEYwWpBhUBRlJwEikwIOzYdYWNPy5JMl1riNFwr1GaiHnUE6zoXK3qLxQhWCyoEiqLkJODzFJxHMDIdqVugGBazdwoVlTlZQ7V0Da11tXaoZgbx6YQKgaIoOSklRlCv+ABYWUNAwaIyx2KpZdbQWpdFUO5kt9MVFQJFUXLiNF/LRTieZHIhXreMIViMEZRiEdSyjsDp4tpMqBAoipKToC9/jODYdBSoXw0BLF7cC6ePpvB7paZ37rVsaX26UNVPLCK9InK/iOyzv/fkOS7lGly/y7V9i4g8JiL7ReRuEQnker2iKI2nkGuo3qmjsBgsLpY1VMv4QLNSrfTdAjxojNkGPGg/z0XEGLPD/rrOtf2vgc8ZY84CpoD3VrkeRVFqRCEhOJ4Rgvr1zM+4hopkDdUyY8jhXa8e5LpXbaj5+56qVJsbdT3wBvvxncAPgA+X8kKxcs7eBPya6/WfAL5Q5ZoURakBAa8nM0Q9m5lIAoDu1voZ8RnXUIGAdTyZrosQ/M27XlXz9zyVqfYMrjXGHLcfnwDW5jkuJCK7ReRREbnB3tYHTBtjnL+0o8DGfB8kIjfb77F7bGysymUrilKMQhbBQsxu01zH9Eqfp4RgcTJ92rd3OBUoahGIyANArg5IH3U/McYYEcn3G9tsjBkRka3AQyLyDDBTzkKNMbcDtwMMDw8X7k2rKErVBHzevEIwH0vRHvTVNb2y1S7miiby9zuKJTRGUAuKCoEx5qp8+0TkpIisN8YcF5H1wGie9xixvx8UkR8AFwPfArpFxGdbBYPASAU/g6IodSDgzV9QthBL0lbn9gtOe4eFeG73FNgxghrWEDQr1Z7BXcBN9uObgO9kHyAiPSIStB/3A68FnjPGGOBh4J2FXq8oysoQKJA+Oh+v32QyB6e9QzhewCKoU4yg2aj2DN4KXC0i+4Cr7OeIyLCI3GEfsx3YLSJPYV34bzXGPGfv+zDwJyKyHytm8C9VrkdRlBoRLBAjmI/Wb2i9Q8YiiOW3COKaPloTqvpNGmMmgDfn2L4beJ/9+MfAhXlefxDYWc0aFEWpD8WCxXUXAjsQXcwi6GypT/fTZkJtKkVRclKo+2g9h9Y7+Lwegj5P8RiBuoaqRs+goig5Cfg8pNKGVI4B8gvx+lsEAG1BH+FYfotgfD5e11qGZkGFQFGUnAQKDLBfiKXqnjUE1qjIfBbBbDTB5EKczX2tdV/HakeFQFGUnBQaYD8fS9IerL9vvi2Q3yI4PBEGYEiFoGpUCBRFyYljEcSyBtjHk2niyTTtjbAIgosWwUwkwf3PncTKPIdXbCE4o7et7utY7agQKIqSk3yuISeds97BYrAtAjtr6O8f3Mfv/NtufnxgAoCXJxYAOEMtgqpRIVAUJSfBPEIw30AhaA14M8LjzCW4+4kjAByZDNPXFmhI0Hq1o0KgKEpOMjGCrBRS5w69EYPd24KLFsGCHSv42ZEpAEbnYkvGSiqVo0KgKEpO/HmCxbGkdUEONaDHT2vAS9iOEYzPxwCrqhlgbC7Gms7mGytZD1QIFEXJSUvACgZHsip7nUZ0jWjt0Bb0ZSyBsTlbCGJJjDGMzcUYaFchqAUqBIqi5KTHLtSaCseXbHcshEZ0/WwNeIkkUqTSJmMRJFKGaCLN+HyMgSYcNF8PVAgURclJX7slBOPzS4XAcQ0FGjDkfaM9E/nA2DwTC3E67MDwkakwybRRIagRKgSKouTEsQgmF7KEINE4i+DVm3sAePD5UVJpw1C/VTNwcMxKHV3TocHiWqBCoChKTgI+Dx0h33IhaGCMYEt/Gz2tfu7da03EdYTg0LglBGoR1AYVAkVR8tLXFmBiIXeMoBGzgkWE4aFenj5qTbZ9zZZeAA6NzwPQ364N52qBCoGiKHnpaw8yuRBbss2JETSq/fPV29dmHr9qsBuAkekIsOi+UqpDhUBRlLz0tgWYWBYsdlxDjbl8XHWeJQSDPS10hKxg8bHpKEDmuVIdehYVRclLX1uAPUeml2yLNdA1BJYY3fGbw5y9tiMzvnJkOkJbwIuvAZlLzYAKgaIoeRnoCDK5ECcST2UKzDJC0MCLsGMVRBOWWyqeTNPXpRlDtaKq36SI9IrI/SKyz/7ek+OYN4rIHtdXVERusPd9WUQOufbtqGY9iqLUlkuHekmlDT85OJ7Z5oyHFJGGryfo8+D3Wp/bGdJZxbWiWkm/BXjQGLMNeNB+vgRjzMPGmB3GmB3Am4Aw8H3XIR9y9htj9lS5HkVRashrtvbSGvDy0AujmW2xRHrF5gSLSKbbaGeLOjRqRbW/zeuBO+3HdwI3FDn+ncC9xphwlZ+rKEoDCPq8XHJGD8/Y6ZtguYYCDaghyEe/3V9ILYLaUa0QrDXGHLcfnwDWFjoYuBH4eta2T4vI0yLyORHR6hBFOcVY2xnKNHwDyz+/UhYBwBm91iCazhYVglpR9LcpIg+IyN4cX9e7jzPW/DhT4H3WAxcC97k2fwQ4F7gU6AU+XOD1N4vIbhHZPTY2VmzZiqLUiIGOIGPzscyIyFgy1ZD2EvnY5AiBpo7WjKJn0hhzVb59InJSRNYbY47bF/rRfMcCvwz8pzEm4Xpvx5qIici/Ah8ssI7bgdsBhoeH8wqOoii1ZU1HkETKMB1O0NMWsFxDK5i2OdhjNaJLGb0M1Ipqf5u7gJvsxzcB3ylw7K+S5RayxQOx0g9uAPZWuR5FUWqM089nzG4DHU+mCfpXLkbgdEWdDieKHKmUSrVCcCtwtYjsA66ynyMiwyJyh3OQiAwBm4BHsl7/7yLyDPAM0A98qsr1KIpSY9bYQjA6awmBkz66UmzstlxD2l6idlTlZDPGTABvzrF9N/A+1/OXgY05jntTNZ+vKEr9WbQIrLYOsWR6RQfGXzrUw9/duIOrthfLTVFKRaMtiqIUZCDbIkik6WtbOYtARLh+x7L7SqUKtFGHoigFaQ/6CPo8mXbU8VS6IbMIlMahQqAoSkFEhJ7WAFO2EMSSqYY1nFMag/42FUUpSnern+mIlaWzki0mlPqgv01FUYrS3epnOuxYBGm1CFYZ+ttUFKUoPa0Bpuy8/UgiRcsK1hEotUeFQFGUolgWQYJU2hBPpjOzCZTVgQqBoihF6W4NMB2OE44nAWhVIVhVqBAoilKUnlY/ybRh3J5f3BLQEqTVhAqBoihF6W6x2jkcn44AaIxglaFCoChKUbpbrd7/I7YQqGtodaFCoChKUXrabItgxuo3pMHi1YUKgaIoRem2p4Edn7EtAnUNrSpUCBRFKUq33fJ5ZFotgtWICoGiKEVxYgTHNEawKlEhUBSlKH6vh/agbzFrSNNHVxUqBIqilER3q5+FeArQGMFqQ4VAUZSScI+G1BjB6kKFQFGUknDiBCJoG+pVhv42FUUpCSdzqNXvRURWeDVKLalKCETkXSLyrIikRWS4wHHXisiLIrJfRG5xbd8iIo/Z2+8WkUC+91AUZWVxagk0ULz6qNYi2Au8A/hhvgNExAvcBrwVOA/4VRE5z97918DnjDFnAVPAe6tcj6IodaLHdg151Y+w6qjqV2qMed4Y82KRw3YC+40xB40xceAu4HqxbMs3Ad+0j7sTuKGa9SiKUj92bukD4ORsbIVXotSaRth4G4EjrudHgdcAfcC0MSbp2r4x35uIyM3AzQBnnHFGfVaqKEpeXretn7+7cQfxZHqll6LUmKJCICIPAOty7PqoMeY7tV9SbowxtwO3AwwPD5tGfa6iKItcvyPvvZpyGlNUCIwxV1X5GSPAJtfzQXvbBNAtIj7bKnC2K4qiKA2kEWGfJ4BtdoZQALgR2GWMMcDDwDvt424CGmZhKIqiKBbVpo/+oogcBS4H/ktE7rO3bxCRewDsu/33A/cBzwPfMMY8a7/Fh4E/EZH9WDGDf6lmPYqiKEr5iHVjfnoxPDxsdu/evdLLUBRFOa0QkSeNMctqvjQjWFEUpclRIVAURWlyVAgURVGaHBUCRVGUJue0DBaLyBjwSoUv7wfGa7icWqHrKo9TdV1w6q5N11Ueq3Fdm40xA9kbT0shqAYR2Z0rar7S6LrK41RdF5y6a9N1lUczrUtdQ4qiKE2OCoGiKEqT04xCcPtKLyAPqzOeXQAABNRJREFUuq7yOFXXBafu2nRd5dE062q6GIGiKIqylGa0CBRFURQXKgSKoihNTlMJgYhcKyIvish+Ebllhdfysog8IyJ7RGS3va1XRO4XkX32954GrONLIjIqIntd23KuQyz+3j5/T4vIJQ1e1ydEZMQ+Z3tE5G2ufR+x1/WiiFxTx3VtEpGHReQ5EXlWRP7I3r6i56zAulb0nIlISEQeF5Gn7HX9hb19i4g8Zn/+3XaLekQkaD/fb+8favC6viwih1zna4e9vWF/+/bneUXkZyLyPft5fc+XMaYpvgAvcADYCgSAp4DzVnA9LwP9Wds+A9xiP74F+OsGrONK4BJgb7F1AG8D7gUEuAx4rMHr+gTwwRzHnmf/PoPAFvv37K3TutYDl9iPO4CX7M9f0XNWYF0res7sn7vdfuwHHrPPwzeAG+3t/wz8D/vx7wP/bD++Ebi7Tucr37q+DLwzx/EN+9u3P+9PgK8B37Of1/V8NZNFsBPYb4w5aIyJA3cB16/wmrK5HrjTfnwncEO9P9AY80NgssR1XA/8m7F4FGvC3PoGrisf1wN3GWNixphDwH6s33c91nXcGPNT+/Ec1oyNjazwOSuwrnw05JzZP/e8/dRvfxngTcA37e3Z58s5j98E3iwi0sB15aNhf/siMgi8HbjDfi7U+Xw1kxBsBI64nh+l8D9KvTHA90XkSRG52d621hhz3H58Ali7MkvLu45T4Ry+3zbNv+Ryna3Iumwz/GKsu8lT5pxlrQtW+JzZbo49wChwP5b1MW2soVXZn51Zl71/BmtoVd3XZYxxzten7fP1OREJZq8rx5przeeBPwPS9vM+6ny+mkkITjVeZ4y5BHgr8AcicqV7p7FsvRXP7T1V1mHzBeBMYAdwHPjblVqIiLQD3wI+YIyZde9byXOWY10rfs6MMSljzA6sueQ7gXMbvYZcZK9LRC4APoK1vkuBXqwpig1DRH4eGDXGPNnIz20mIRgBNrmeD9rbVgRjzIj9fRT4T6x/kJOOuWl/H12h5eVbx4qeQ2PMSfufNw38fyy6Mhq6LhHxY11s/90Y87/szSt+znKt61Q5Z/ZaprHmlF+O5Vrx5fjszLrs/V3ARIPWda3tYjPGmBjwrzT+fL0WuE5EXsZyX78J+DvqfL6aSQieALbZ0fcAVmBl10osRETaRKTDeQy8Bdhrr+cm+7CbgO+sxPoKrGMX8Jt2BsVlwIzLHVJ3snyyv4h1zpx13WhnUGwBtgGP12kNgjVb+3ljzGddu1b0nOVb10qfMxEZEJFu+3ELcDVW/OJh4J32YdnnyzmP7wQesi2sRqzrBZeYC5Yf3n2+6v57NMZ8xBgzaIwZwrpGPWSMeTf1Pl+1jHSf6l9Ykf+XsHyUH13BdWzFyth4CnjWWQuWb+9BYB/wANDbgLV8HctlkMDyPb433zqwMiZus8/fM8Bwg9f1Fftzn7b/Ada7jv+ova4XgbfWcV2vw3L7PA3ssb/ettLnrMC6VvScARcBP7M/fy/wcdf/wONYQer/AIL29pD9fL+9f2uD1/WQfb72Al9lMbOoYX/7rjW+gcWsobqeL20xoSiK0uQ0k2tIURRFyYEKgaIoSpOjQqAoitLkqBAoiqI0OSoEiqIoTY4KgaIoSpOjQqAoitLk/B+0GJb/klZm8AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "THF9GD2jjiaz"
      },
      "source": [
        "### (STFT) "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nSn7CGFWh0Wc",
        "outputId": "3df7be9e-19d5-418b-c344-3b29bd808f6a"
      },
      "source": [
        "features[1].numpy()[0][:,:,0].shape"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(22, 33)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CdVUyXd9k5Tr"
      },
      "source": [
        "### (Magnitude) "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 292
        },
        "id": "eMlPS3HVlPn_",
        "outputId": "212ba8ca-1d31-41f2-b887-b2c577643e50"
      },
      "source": [
        "plt.matshow(features[2].numpy()[0][:,:,0])"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7fbc9bb97190>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAECCAYAAADn84z1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQVUlEQVR4nO3db4xc9XXG8efZ9a7XNg62ZeI6hkBDUVuUtiRa0X+ooiWJaKQWIrWoSK1cKZLzIkhEbaWivAmtVIlWTdq8qKicguKoCZQGKKhCbShCopEqGkMJGJwEGhkF13hjCGBisNnd0xd7UbbbXc+Zmbszc4bvR7J25s7Pd36/vetnr+/MmeOIEACgnolhTwAA0BsCHACKIsABoCgCHACKIsABoCgCHACKGmqA277a9rdtP2f7pmHOpR+2j9h+yvYTtg8Oez7dsH277Tnbh5Zt22H7QdvPNl+3D3OOWWus5WbbR5tj84Ttjw5zjhm2L7D9sO1nbD9t+8Zme7njcpa1VDwuM7b/0/Y3m7X8SbP9x20/2uTYP9ieHtichvU+cNuTkr4j6cOSXpD0DUnXR8QzQ5lQH2wfkTQbESeGPZdu2f4VSa9L+lJEvL/Z9heSXo6IW5pfrNsj4o+HOc+MNdZys6TXI+Ivhzm3btjeLWl3RDxue6ukxyRdK+n3Vey4nGUt16necbGkLRHxuu0pSV+XdKOkP5B0T0TcaftvJX0zIm4dxJyGeQZ+uaTnIuK7EXFG0p2SrhnifN6RIuIRSS+v2HyNpAPN7QNa+gc38tZYSzkRcSwiHm9un5R0WNIeFTwuZ1lLObHk9ebuVPMnJP2apK822wd6XIYZ4HskfW/Z/RdU9MBq6SB+zfZjtvcNezIt2BURx5rbL0raNczJtOAG2082l1hG/rLDcrYvkvQBSY+q+HFZsRap4HGxPWn7CUlzkh6U9N+SXomI+WbIQHOMFzHbcUVEfFDSr0v6ZPNf+bEQS9fYKn/ewq2SLpZ0maRjkj473Onk2T5H0t2SPhURry1/rNpxWWUtJY9LRCxExGWSztfSVYSfGuZ8hhngRyVdsOz++c22ciLiaPN1TtK9WjqwlR1vrl2+fQ1zbsjz6VlEHG/+0S1K+oKKHJvmGuvdkr4cEfc0m0sel9XWUvW4vC0iXpH0sKRflLTN9obmoYHm2DAD/BuSLmlewZ2W9DuS7h/ifHpie0vz4oxsb5H0EUmHzv63Rt79kvY2t/dKum+Ic+nL24HX+JgKHJvmxbLbJB2OiM8te6jccVlrLUWPy3m2tzW3N2npDRiHtRTkv9UMG+hxGdq7UCSpeevQX0ualHR7RPzZ0CbTI9vv09JZtyRtkPSVSuuwfYekKyXtlHRc0mck/ZOkuyS9V9Lzkq6LiJF/cXCNtVyppf+mh6Qjkj6x7DrySLJ9haR/l/SUpMVm86e1dO241HE5y1quV73j8rNaepFyUksnv3dFxJ82GXCnpB2S/kvS70bE6YHMiY+TBYCaeBETAIoiwAGgKAIcAIoiwAGgKAIcAIoaeoCPSem5pPFZy7isQ2Ito2pc1jLsdQw9wCWNxYFsjMtaxmUdEmsZVeOylnd8gAMAejDQQp5pb4wZbfk/297SaU1pY287tFuY1Qp9fD/6WssIGZd1SKxlVI3LWga1jpP6wYmIOG/l9g2rDc6yfbWkz2uptPTvIuKWs42f0Rb9vK/K7Dj3/BumUuO6EfNvJQcOqYI1+0uLCltgbPxbfPX51bb3fAml6ajzN1r6CNVLJV1v+9Je9wcA6E4/18DpqAMAQ9RPgI9TRx0AKKeva+AZzfsk90nSjDav99MBwDtGP2fgqY46EbE/ImYjYnYcXnUGgFHRT4CPRUcdAKiq50soETFv+wZJ/6ofddR5urWZAQDOqq9r4BHxgKQHWppLDxNY7Dym630m3z+9HkVEGby/G0CDUnoAKIoAB4CiCHAAKIoAB4CiCHAAKIoAB4CiCHAAKIoAB4CiCHAAKGrdP42wJ8lqw1hYyO3PXfyempjMjctWgbZd2dl2BSiVnUBZnIEDQFEEOAAURYADQFEEOAAURYADQFEEOAAURYADQFEEOAAURYADQFEEOAAUNZql9FnZMvBIltx3o+3S91Fvpgxg5HAGDgBFEeAAUBQBDgBFEeAAUBQBDgBFEeAAUBQBDgBFEeAAUBQBDgBFDb4Ss81Kwm6aFWdlmxUDwJBxBg4ARRHgAFAUAQ4ARRHgAFAUAQ4ARRHgAFAUAQ4ARRHgAFAUAQ4ARQ2+EjPT+zFZremJ9vtDtt4+k16XANYJZ+AAUFRfZ+C2j0g6KWlB0nxEzLYxKQBAZ21cQvnViDjRwn4AAF3gEgoAFNVvgIekr9l+zPa+NiYEAMjp9xLKFRFx1Pa7JT1o+1sR8cjyAU2w75OkGW3u8+kAAG/r6ww8Io42X+ck3Svp8lXG7I+I2YiYndLGfp4OALBMzwFue4vtrW/flvQRSYfamhgA4Oz6uYSyS9K9XipA2SDpKxHxL63MCgDQUc8BHhHflfRzLc5l+c5zwxZz47qq2Mz22Rz13pnZClAAZfE2QgAoigAHgKIIcAAoigAHgKIIcAAoigAHgKIIcAAoigAHgKIIcAAoavA9MduUrIbsqs9l2xWMbfe6zFaKKlkpSsUmUBZn4ABQFAEOAEUR4ABQFAEOAEUR4ABQFAEOAEUR4ABQFAEOAEUR4ABQVO1KzCRPTqbHZvtspntiZisdsxWbbT8vgLI4AweAoghwACiKAAeAoghwACiKAAeAoghwACiKAAeAoghwACiKAAeAoghwAChqNEvpk2Xl6RL5dCNgSUp2QM7us6uOygCQxxk4ABRFgANAUQQ4ABRFgANAUQQ4ABRFgANAUQQ4ABRFgANAUQQ4ABQ1mpWYWelqyGQjYElazFZiJpsQZ9GEGECXOAMHgKI6Brjt223P2T60bNsO2w/afrb5un19pwkAWClzBv5FSVev2HaTpIci4hJJDzX3AQAD1DHAI+IRSS+v2HyNpAPN7QOSrm15XgCADnq9Br4rIo41t1+UtKul+QAAkvp+ETMiQtKab6Gwvc/2QdsH39Lpfp8OANDoNcCP294tSc3XubUGRsT+iJiNiNkpbezx6QAAK/Ua4PdL2tvc3ivpvnamAwDIyryN8A5J/yHpJ22/YPvjkm6R9GHbz0r6UHMfADBAHSsxI+L6NR66quW5LH/S1DBP5v4DEdFF1eQCPSwB1EAlJgAURYADQFEEOAAURYADQFEEOAAURYADQFEEOAAURYADQFEEOAAUNfiemBOTnYdsmsntaus5qXFx6o3UOElaXGy5CnR+Pjeu7QrQ9egXmu3b2Xa/0Cz6iuIdhjNwACiKAAeAoghwACiKAAeAoghwACiKAAeAoghwACiKAAeAoghwAChq8JWYico/X7gntasfXnhuatzM3KnUOEmaePZ7qXHZCktvaPlbnKywjPm3crub7FwZ26221xwLuWrR7Jq7e3KqOzG6OAMHgKIIcAAoigAHgKIIcAAoigAHgKIIcAAoigAHgKIIcAAoigAHgKIIcAAoaqCl9J6Y0MTmzR3HvfIzO1L7O3ZlrsR626Fcyb0kvefEa7mBrybHZZskz2xMjcuWlevN3LBumil7Kvfj4unp3HMnP45A2XHrIdugedQbPq8HPmZg6DgDB4CiCHAAKIoAB4CiCHAAKIoAB4CiCHAAKIoAB4CiCHAAKIoAB4CiBtvUeMKpisNX35f7vfL5D30pNe6Ptv12apwk6Z+T4zbmKid1JtlceEvnClVJ0lu5/cWbp3PPO5GvDMw2QI5khV6cOZMbl6xmXRdtVxsmm1Jnj0v6e5NoJo56Ov402b7d9pztQ8u23Wz7qO0nmj8fXd9pAgBWypwOfFHS1ats/6uIuKz580C70wIAdNIxwCPiEUkvD2AuAIAu9PMi5g22n2wusWxvbUYAgJReA/xWSRdLukzSMUmfXWug7X22D9o+eGYx+RmnAICOegrwiDgeEQsRsSjpC5IuP8vY/RExGxGz0xMzvc4TALBCTwFue/eyux+TdGitsQCA9dHxfeC275B0paSdtl+Q9BlJV9q+TFJIOiLpE+s4RwDAKjoGeERcv8rm29ZhLgCALgy0EjMWFrSY6CW5OJXb329uOZUad/t7XsztUNKpPT+WGrfh299Ljcv2fXS272Oyh2W2f2U3PTGzlZNKVmx6Q3KOyWrWrGxF6XrIVk4OrcKSPpel8FkoAFAUAQ4ARRHgAFAUAQ4ARRHgAFAUAQ4ARRHgAFAUAQ4ARRHgAFDUQCsx7Qk50UtyunOxpiTpz1+6JDXuyUMX5XYo6aeP5ao2Y9fO1DgfP5Hb30Kuom7x5OupcRObcp/8GD9MVoBKqWMnKV0tmu2dme8Pmew3OT2dGteNxWQP0nTlZLYiciJZVUpPzLHEGTgAFEWAA0BRBDgAFEWAA0BRBDgAFEWAA0BRBDgAFEWAA0BRBDgAFDXYnpgRqf6GE8nWi7cd+qXUuJ/4+zdzO5S0sO2c1Lg3d29OjYuLt6XGzXw/N8fJl7amxvlUbn8TU11UJZ7OVRsuJntYxnxuXLaHZbYPaLYXp5TvaarFZG9R56pKs+PSVar51qd52bVk0Y+za5yBA0BRBDgAFEWAA0BRBDgAFEWAA0BRBDgAFEWAA0BRBDgAFEWAA0BRg+2JOTmpiW3ndhx3entuf+dufSM17o3dyR1K2nr45dS4+U25is35TblqtY0nkhV1/3M8Nc47d6TGLc7lenZKkqen0mPbFIvJ3plah3LDdBVorqI1kv1CW+9h6eS52nqUbFJhuW44AweAoghwACiKAAeAoghwACiKAAeAoghwACiKAAeAoghwACiKAAeAoghwAChqoKX0mpyUtr+r47Az23NlxL+x5zupcfe/P9f8WJLe9ViuGfDMS7mGvBu/dTQ1bmHPztQ4n787NS5e+kFuf5Nd/A7PloEnG+1mmxVny8CzTZLT5exSugy8q31mZNec/JiB/PO23KgY66rjT4ntC2w/bPsZ20/bvrHZvsP2g7afbb7mP3AEANC3zK/5eUl/GBGXSvoFSZ+0famkmyQ9FBGXSHqouQ8AGJCOAR4RxyLi8eb2SUmHJe2RdI2kA82wA5KuXa9JAgD+v65exLR9kaQPSHpU0q6IONY89KKkXa3ODABwVukAt32OpLslfSoiXlv+WESEpFVfTbG9z/ZB2wfPLJzqa7IAgB9JBbjtKS2F95cj4p5m83Hbu5vHd0uaW+3vRsT+iJiNiNnpyc1tzBkAoNy7UCzpNkmHI+Jzyx66X9Le5vZeSfe1Pz0AwFoy7wP/ZUm/J+kp20802z4t6RZJd9n+uKTnJV23PlMEAKymY4BHxNclrfXu/qvanQ4AIGuglZhx+rQWnjvScdzkG+9O7e8fn/xgatzUVL5a7cx7cxWRG59/KbfDTTOpYYszuYbBE2/kqg0XXz2ZGtdNBaGnkj8uLVcHpqtFJ3KNhbtpshtnzuQGLia/j9lKRxoBI4HPQgGAoghwACiKAAeAoghwACiKAAeAoghwACiKAAeAoghwACiKAAeAohwDrPiy/X0tfW7KcjslnRjYJNbXuKxlXNYhsZZRNS5rGdQ6LoyI81ZuHGiAr8b2wYiYHeokWjIuaxmXdUisZVSNy1qGvQ4uoQBAUQQ4ABQ1CgG+f9gTaNG4rGVc1iGxllE1LmsZ6jqGfg0cANCbUTgDBwD0gAAHgKIIcAAoigAHgKIIcAAo6n8BugyDu6zPSvEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PQqJxBiFk730"
      },
      "source": [
        "### (Magnitude to decibel) "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "Z7xuh9S_jzE5",
        "outputId": "09077300-ab6c-4b7f-8e47-0d5142b0e178"
      },
      "source": [
        "cmin = np.min(features[3].numpy()[0][:,:,0])\n",
        "cmax = np.max(features[3].numpy()[0][:,:,0])\n",
        "plt.imshow(features[3].numpy()[0][:,:,0])"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7fbcb61779d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAD4CAYAAAAaT9YAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAezklEQVR4nO3de4xc93Uf8O93HvvkLpcv8S1RsmUpkqtHwNBx4qbyS5EEI7ILxxXROnLrgE5gAzaSoFVTwHIdFHAfdoJEgRXGIiy3jqQ0tmwBUWwRqltZri2LlimJEvWgZMXkkuKSXC73vbOzc/rHXqKr5Qzvlztj7f7I7wcgdubO2Xt/9zFnLu/cs4cRATMzS1NhsQdgZmYL5yRuZpYwJ3Ezs4Q5iZuZJcxJ3MwsYaXFHkA9pc7uKC9fmRtX69DurCmVZ6S4AKW4mYr22ceZ/PkVKtKsEEUtrtamxaGmhVHbdChUtbjQNjGirMWp6yHuWoSwa9Vtosa1/FRqEbYJoB8DMnV8LZwXxW1XnNRyT5S0BSvvi8rYIKYnx86IXJJJvLx8JS77nT/IjRu9UsuAq9cNS3HVGe1oHTrcK8WVT+Zn3u5D2k6uLpPCMHaJljmKY9q6lk9p4+sY1A7qmXZtfhNrxQ/ocfGDt00cX1d+XHlY23ZtQ1IYau1inPhBXpoQ5ye++2c6tLiOE9o2VhOleqwoCVA9CSqNa+uw4mUt90yu0s5GlH2x7x/+rO70ps4BSN5E8kWSB0jeUef1dpIPZK8/QXJLM8szM7M3WnASJ1kE8JcAbgZwFYDtJK+aF/ZxACcj4q0A/hTAf17o8szM7EzNnIlvA3AgIl6NiAqA+wHcOi/mVgD3Zo//DsB7SYpXp8zMLE8zSXwjgINznh/KptWNiYgqgFMAVtWbGckdJPeQ3DMzMdbEsMzMLhxL5hbDiNgZEVsjYmuxs3uxh2NmloRmkng/gM1znm/KptWNIVkCsBzAiSaWaWZmczSTxJ8EcDnJS0m2AbgNwEPzYh4CcHv2+MMA/lf4zyaambXMgu8Tj4gqyU8B+C6AIoBdEfEcyc8D2BMRDwG4B8B/J3kAwCBmE72ZmbVIU8U+EfEwgIfnTfvsnMeTAH67mWWclVARCQCjE2I1hWjtD7T/wBz9jfwStqEV4s06BbH45ZS2S0uj2nLbT0lhmFLXQ1SoaPMra3VciOXa/GpCrVR5VFumWiWoFvGoBVVaCSNQEbeJur6VXm1+hWltfmqBzoxQpawWQBXEKttaWcsBPa+MSHHFo/mVYaWR+gVGS+aLTTMzO3dO4mZmCXMSNzNLmJO4mVnCnMTNzBLmJG5mljAncTOzhDmJm5klzEnczCxhS7I9W60NGN+U38Op7xmt9dHQ9VoJW6FdK9cauFFsjClUlHJCW4fo0RoYVldr5XDVHu3ze7pXi+vbr1XrTa7W4roPa/tsYo3Y3k5ouwYAUcyPq4l9E9vEVl8Q/8S+2rOz46TW/6zarZVEypWTYhs3dX4qCpt52WFtm0yKlcdTfdpKFCraRmFVaPl4vP4yfSZuZpYwJ3Ezs4Q5iZuZJcxJ3MwsYU7iZmYJcxI3M0vYgpM4yc0kv0fyeZLPkfx0nZgbSJ4iuTf799l68zIzs4Vp5j7xKoA/jIinSPYA+AnJ3RHx/Ly470fEB5pYjpmZNbDgM/GIOBIRT2WPRwDsB7CxVQMzM7N8LanYJLkFwPUAnqjz8jtJPg3gMIA/iojnGsxjB4AdAFDuWYHeA/mfL6eu1ErYygNaVWS1t7WlZOwVqidLWlUfx8TqujatMk1ebk3sY3qJttii2OtwcpVYiblMrMRU11eosq0JPR0BoPO4ti/G12j7Vq3YnBKrbKstrrAsiIXMbVrbSXBG22fT3fn7bHSDtk06j2vLnGlTK5S13FMQyl2jWH+ZTX+xSXIZgG8A+ExEzG9b+xSASyLiWgB/AeBbDQcYsTMitkbE1lJnd7PDMjO7IDSVxEmWMZvAvx4R35z/ekQMR8Ro9vhhAGWSq5tZppmZ/X/N3J1CAPcA2B8RX2oQsy6LA8lt2fJOLHSZZmb2Rs1cE/91AB8F8CzJvdm0PwZwMQBExN0APgzg90lWAUwAuC0ixD/tZmZmeRacxCPicQBnvbofEXcBuGuhyzAzs7NzxaaZWcKcxM3MEuYkbmaWMCdxM7OELckemzOdgaF/kl/tyA6thG2mT+yxKX6kzYyLm21CKHVTGgQCYGgVYmf/qnkOoTIRADqOaXE1cZMU1fakYlWkWsXY/brYU7QnP6Y4qS2zsqy150hTYv/HmXZtfiEOr3NAO0a7jmkVqmPrWrtd1jyTf1CduEo7oCo92jYuVrRtUhKPlfLr+WWsrNbfvj4TNzNLmJO4mVnCnMTNzBLmJG5mljAncTOzhDmJm5klzEnczCxhTuJmZglzEjczS9iSrNgsTBC9+/N70w1fq1WIdfeKZVOi0Umx6WA1/zOyMKl9jta6tHVVq1hDXIeJddpyS6NqRaRYPTutVc695c9fkeJ+9om3SnFKL87ilFjFqrVXRO/BqhQ3ul7bZ7WyNr6CWO3a+3NtfOVhrRx3bG2XFKeOb+D6/GpMsTBa7hOqKla098/MiwdyYyKm6k73mbiZWcJa0Sj5NZLPktxLck+d10nyz0keIPkMyV9udplmZjarVZdT3h0Rxxu8djOAy7N/7wDw5eynmZk16c24nHIrgK/FrB8B6CO5/k1YrpnZea8VSTwAPELyJyR31Hl9I4CDc54fyqa9AckdJPeQ3DMzMdaCYZmZnf9acTnlXRHRT/IiALtJvhARj53rTCJiJ4CdANC5drP4XbKZ2YWt6TPxiOjPfg4AeBDAtnkh/QA2z3m+KZtmZmZNaiqJk+wm2XP6MYAbAeybF/YQgN/J7lL5VQCnIuJIM8s1M7NZzV5OWQvgQZKn5/U3EfEdkr8HABFxN4CHAdwC4ACAcQD/Om+mtTZgbHP+TfLF41rLpcl2rWqgWNLiCm1aXG06/zOy70WtMOPkr2kFF1HRPpfLg9qur4pFRlEQr4CpXebE2R38qFbEo7aFmxYOKbVTnlp0M7Vc22cr92tFa8ev6ZTippdJYTh5uVa1FAUtrqrV+qCmtugT3hrlUW1ey17X3ttTPdo+W/bSkBQ3fsuv5MbE4z+sO72pJB4RrwK4ts70u+c8DgCfbGY5ZmZWnys2zcwS5iRuZpYwJ3Ezs4Q5iZuZJcxJ3MwsYU7iZmYJcxI3M0uYk7iZWcKWZHu2YgXo+Vn+50tNHP3wSrE6sapVnBVHtTZZnYP56zB4nVaJKRYJoiC2Z5sWqwTLQ9q6lse0EarVjh0ntJLNarc2w+l2bblKS7UQT33UykTWtHU4cXWHFFfp1ZbbflLcxl3q0adR266FGFcW/ujpsiNa5fHIRu14pzi2Y+9YJcWVx/P3RRTq7wefiZuZJcxJ3MwsYU7iZmYJcxI3M0uYk7iZWcKcxM3MErbgJE7yCpJ75/wbJvmZeTE3kDw1J+azzQ/ZzMxOW/B94hHxIoDrAIBkEbN9Mx+sE/r9iPjAQpdjZmaNtepyynsBvBIR/9ii+ZmZmaBVFZu3AbivwWvvJPk0gMMA/iginqsXRHIHgB0AUO5dgapQnDbdq1WcsaRVa3Us1xoxVnq0zTaxIj+uKPa6nKG4q7q0CtDiuPb53TakVetVlmv7ou2UNj+1SlA5TgCg2qONb6YjP25arIjse1GLUyuPp5dp26QmVqdOrNPmx2ltfl2va9u4NKnFTa0Qj9Hh/PlNrNTWteOkliuqnWKlsLjPKr35cTMNqombPhMn2QbgtwD8zzovPwXgkoi4FsBfAPhWo/lExM6I2BoRW4ud3c0Oy8zsgtCKyyk3A3gqIo7OfyEihiNiNHv8MIAyydUtWKaZmaE1SXw7GlxKIbmOJLPH27LlnWjBMs3MDE1eEyfZDeD9AD4xZ9rvAUBE3A3gwwB+n2QVwASA2yJCuyBmZma5mkriETEGYNW8aXfPeXwXgLuaWYaZmTXmik0zs4Q5iZuZJcxJ3MwsYU7iZmYJW5I9NhFAQSg8nFqjNbrr7pmS4ooFrVpruqJttkI5f3xRau0uiHFtfuVT2ud3rU1bbmlCrCYUV1ftY1nUdi1iVKwA7RR6HWptGDG2Xuw7Kq6rutyO41rcxBotriZsEwCo9InHgNATczZQC6v05C9X3sZiO9Geg1pl9MhG7YAPIYwNdoPPxM3MEuYkbmaWMCdxM7OEOYmbmSXMSdzMLGFO4mZmCXMSNzNLmJO4mVnCnMTNzBK2JCs2GUBxSqgSK2mVZJ1tWpPA8akGTezmmTmmNXaM9vySs6JWdIrChPZ5S7HKTY1rG9bi1GpCtQpPPb2YEStKi1r7VBQq+SV7JbH6szSpLbOyXIurFbXjXe1PWhYrJ9X5qdTtEoXWtR6YWCNWk5bVOC11rn20X4obu3ptbkyxUn97+EzczCxhUhInuYvkAMl9c6atJLmb5MvZzxUNfvf2LOZlkre3auBmZqafiX8VwE3zpt0B4NGIuBzAo9nzNyC5EsCdAN4BYBuAOxslezMzO3dSEo+IxwAMzpt8K4B7s8f3AvhgnV/9TQC7I2IwIk4C2I0zPwzMzGyBmrkmvjYijmSPXwdQ78r8RgAH5zw/lE07A8kdJPeQ3FOdUP9WpZnZha0lX2xmHeyb+io5InZGxNaI2Frq7G7FsMzMznvNJPGjJNcDQPZzoE5MP4DNc55vyqaZmVkLNJPEHwJw+m6T2wF8u07MdwHcSHJF9oXmjdk0MzNrAfUWw/sA/BDAFSQPkfw4gC8AeD/JlwG8L3sOkltJfgUAImIQwJ8AeDL79/lsmpmZtYBUdhQR2xu89N46sXsA/O6c57sA7DqXQc10AENX5Zf2lY9rVVPtm7V+eG0lLW68q0uKa9gUbw61Qqw0Jv6nSSyuq3apfRO1uI5j2vjUXodKj9Vzmd/0Mm09KFTQFrQCYKz/36ekuCM3aCWbU+LNuQWx7+jy17SNPPhL2vtM2XYAUB7X9sWk2LNTqShVjyd136pxlUtWSXGl8fyN16jK2hWbZmYJcxI3M0uYk7iZWcKcxM3MEuYkbmaWMCdxM7OEOYmbmSXMSdzMLGFO4mZmCVuaPTbLNZQvmsiNu/TqE9L8LuvR4gYml0lxk+u0zdZZzi8TOznWKc1r/Jj2lx3ZoZXNFdu0uJlRre9oRehNCQDTF4mlbmpbx5oW2NGvrUetnF9NqPYnPfprWiWmWmFZmNLWtSJWdp5s147jmpglaj1a3PAW7dyxIPZFVfT8XHxfNOhjOZ9aaV0c1lYinn4hP2hmvO5kn4mbmSXMSdzMLGFO4mZmCXMSNzNLmJO4mVnCnMTNzBKWm8RJ7iI5QHLfnGn/leQLJJ8h+SDJvga/+xrJZ0nuJbmnlQM3MzPtTPyrAG6aN203gLdHxDUAXgLw78/y+++OiOsiYuvChmhmZo3kJvGIeAzA4Lxpj0TE6UqWH2G2i72Zmb3JWlGx+W8APNDgtQDwCMkA8FcRsbPRTEjuALADAIor+1A52ZG74Hdf9ZI0wEvbB6S4Y929Uly1dpUUN1HNrxIMrRAThYJWSVYqaOWEQ8e06tT2vkkpru0irYnhuk5tfgWhPykAnJrIP04A4FS71hc1RoW3RBSleU2ulsLQcUKr/msb1uY3sVbsYSmOr/OoNr4psR9r12FtuQWxZ+fU8vzxTS3Xvv5rG9XWQRXt2rFSWrsmN4bH6h+bTSVxkv8BQBXA1xuEvCsi+kleBGA3yReyM/szZAl+JwC0X7KptVvSzOw8teC7U0h+DMAHAPzLiKibdCOiP/s5AOBBANsWujwzMzvTgpI4yZsA/FsAvxURdf8qC8lukj2nHwO4EcC+erFmZrYwyi2G9wH4IYArSB4i+XEAdwHowewlkr0k785iN5B8OPvVtQAeJ/k0gB8D+PuI+M4vZC3MzC5QudfEI2J7ncn3NIg9DOCW7PGrAK5tanRmZnZWrtg0M0uYk7iZWcKcxM3MEuYkbmaWsCXZY7O9YxpXvK0/N+7azp9L87uirPXYPFw6KcX9vHeVFPezsfy47rLWYHF9t1auNzmj9ZJUnTqlVTr2dmuVmCs76vcJnG+s2ibFVWvaeUhMaZVzaMuveK1qrTNRGNfGNrZFLE0Uq1gh9AkFABa16t6RPrHqcEhLJ1MrtArQWisPZapNWzVFsS9qtUtbiZkrN+TGxEj994TPxM3MEuYkbmaWMCdxM7OEOYmbmSXMSdzMLGFO4mZmCXMSNzNLmJO4mVnCnMTNzBK2JCs2y4UZXNQ5khu3rqhVMW4qdWrLpVZNqFpezq9iVCsT13eckuJGZ9qluMPDWj/R9s5pKa67rSLFVUM7b3h9uEeKGx/V1hdqw79q/vg6joiViWu0Ssxij7aN29q1uEpFG19tWjyHK2kbL8Q+sF1ay1sMX6ZVWXYcF4K04lQs/5l2HJ+6THvfjlysxXUM5R8rtQaFs0pTiF0kB0jumzPtcyT7s4YQe0ne0uB3byL5IskDJO/IHaWZmZ0T5aP4qwBuqjP9TyPiuuzfw/NfJFkE8JcAbgZwFYDtJLU28WZmJslN4ll3+sEFzHsbgAMR8WpEVADcD+DWBczHzMwaaOaLzU+RfCa73LKizusbARyc8/xQNs3MzFpkoUn8ywDeAuA6AEcAfLHZgZDcQXIPyT2TQ9qfNTUzu9AtKIlHxNGImImIGoC/xuylk/n6AWye83xTNq3RPHdGxNaI2NrR17GQYZmZXXAWlMRJrp/z9EMA9tUJexLA5SQvJdkG4DYADy1keWZmVl/uDaUk7wNwA4DVJA8BuBPADSSvw+zdt68B+EQWuwHAVyLiloiokvwUgO8CKALYFRHP/ULWwszsApWbxCNie53J9zSIPQzgljnPHwZwxu2HZmbWGkuyYnNsug17+i/OjSus1yrEytR6BHZTu7r0to7Xpbj/M/m23JhxsWJzoqbFVWraLl3RNSHFtZerUtyxkWVS3ESH1pxw9bIxKe7ItLZvp0a0Xocda/K3C9dox12n2BNzZkY77uS4CfFtLVZssqpVThYntbjBt0thKGqHqFSNW6xo+2LwSq0CuOeQ9r4YW6cdnyMb8vdZrVx/+/pvp5iZJcxJ3MwsYU7iZmYJcxI3M0uYk7iZWcKcxM3MEuYkbmaWMCdxM7OELclin/biDLasyv8T5pOh3Ug/XtNaLg3WtB5Or0xeJMVdvexIbsyzI9pf552a0XbVS0NrpLiNy7R2b4Nj2vwuXnFSipuoakU3YxWtuIlafQnKK7Qio3V9Wss/xcGBlVLczLC2TdpOaMd7YYv2V0CjpB3vxaNaAcz0cm1+pTHt3LGgvW0xuUqY17R2oBS0jnoYX6PtC4pt4ZYN5C+40KA7n8/EzcwS5iRuZpYwJ3Ezs4Q5iZuZJcxJ3MwsYU7iZmYJUzr77ALwAQADEfH2bNoDAK7IQvoADEXEdXV+9zUAIwBmAFQjYmuLxm1mZtDuE/8qgLsAfO30hIj4F6cfk/wigLPddPzuiDi+0AGamVljSnu2x0huqfcaSQL4CID3tHZYZmamaLZi858COBoRLzd4PQA8QjIA/FVE7Gw0I5I7AOwAgLY1vTg2lt/ua1qs2BysjUtxj47nt1MDgPVtWrXjhnJ+FeMVHflVnQAwUO2V4i7uzK90BYCT1S4prmetVul4stIpxfWfWi7FDQ92S3EdPdr4Vi7X2r2taM8/Vk5OadsujmmVjljeoBRvnspGrZywvU1rHVaZ1CpFi1oBKGa0XYbpXm09akWxKlLovKZWf7YPapWdfa9ox92pS7VjYKo3/+vJRumu2SS+HcB9Z3n9XRHRT/IiALtJvhARj9Ud4GyC3wkA3ZeLzTPNzC5wC747hWQJwD8H8ECjmIjoz34OAHgQwLaFLs/MzM7UzC2G7wPwQkQcqvciyW6SPacfA7gRwL4mlmdmZvPkJnGS9wH4IYArSB4i+fHspdsw71IKyQ0kH86ergXwOMmnAfwYwN9HxHdaN3QzM1PuTtneYPrH6kw7DOCW7PGrAK5tcnxmZnYWrtg0M0uYk7iZWcKcxM3MEuYkbmaWsCXZY7OzNI1rVh/OjeumVunWJjZiXFcakuL2TWyW4jra88vEyhCb+on+7/HLpLhrVvRLcXuPbZDi1nRrFZGlYmvXtzqtVfUNjWoVpRuE3qM9bVq1Xu+l2vE0NqH1E50e0ar/ymVtG0do74vKcm180aY1lOS4ts+6jmjjm1yTXxvYNqzNqzyq1RmqlZh9r2jlrkOXdUhx9fhM3MwsYU7iZmYJcxI3M0uYk7iZWcKcxM3MEuYkbmaWMCdxM7OEOYmbmSXMSdzMLGFLsmKzvVDFpV3Hc+O6ClovwS5qFWJbylp/yh+Mar04T1Tz+4T+aOSt0ryu7tYqLP/ZmkbtTt/olfE1UlxnWdvGoxWxmrCoVfV1902I89OqE4eO5+8LABhblb8eNWjVf+OTWqWjWFCMUrdWoTwl9s6siRWb6NMaVHJEW277Ce3csdKnVU+2DeWvR0Ersm3Yx3K+GfH0d/gSrRKzOC2sa4MQn4mbmSVM6eyzmeT3SD5P8jmSn86mryS5m+TL2c8VDX7/9izmZZK3t3oFzMwuZMqZeBXAH0bEVQB+FcAnSV4F4A4Aj0bE5QAezZ6/AcmVAO4E8A7MNkm+s1GyNzOzc5ebxCPiSEQ8lT0eAbAfwEYAtwK4Nwu7F8AH6/z6bwLYHRGDEXESwG4AN7Vi4GZmdo7XxEluAXA9gCcArI2II9lLr2O2MfJ8GwEcnPP8UDat3rx3kNxDcs/YSe2LFDOzC52cxEkuA/ANAJ+JiOG5r0VEoOF3p5qI2BkRWyNia/cK7Vt9M7MLnZTESZYxm8C/HhHfzCYfJbk+e309gIE6v9oPYG4HhU3ZNDMzawHl7hQCuAfA/oj40pyXHgJw+m6T2wF8u86vfxfAjSRXZF9o3phNMzOzFlDOxH8dwEcBvIfk3uzfLQC+AOD9JF8G8L7sOUhuJfkVAIiIQQB/AuDJ7N/ns2lmZtYCuRWbEfE40LBE7b114vcA+N05z3cB2HUugzox2Y3/8cKv5MZ9aNtPpfkN1rSqwyK0cq1f6szv/wkAL0zk96dcWdZ6U3aI/URHqVVE/nSg7vfLZxgWe1OuXK6tx3RV28YzYklcX5dW2dmx4aQUV6nlj+/4aLc0r5mDXVpcn3Z8Fka1AusoaV9PlUa0bVzr1o4psZBVroosj2gzbD+Zv74r92vHydDl2vEuvs1QqGr7YuUP8q8yl0br3/Dhik0zs4Q5iZuZJcxJ3MwsYU7iZmYJcxI3M0uYk7iZWcKcxM3MEuYkbmaWMCdxM7OEcfYPEC4tJI8B+Md5k1cDyG+8ubR5HZaO82E9zod1AM6P9Xgz1uGSiDijOe6STOL1kNwTEVsXexzN8DosHefDepwP6wCcH+uxmOvgyylmZglzEjczS1hKSXznYg+gBbwOS8f5sB7nwzoA58d6LNo6JHNN3MzMzpTSmbiZmc3jJG5mlrAln8RJ3kTyRZIHSN6x2ONZKJKvkXw2a2+3Z7HHoyC5i+QAyX1zpq0kuZvky9nPFYs5RkWD9fgcyf55LQeXLJKbSX6P5PMknyP56Wx6MvvjLOuQ2r7oIPljkk9n6/Efs+mXknwiy1UPkGx7U8azlK+JkywCeAnA+wEcwmyfzu0R8fyiDmwBSL4GYGtEJFPUQPI3AIwC+FpEvD2b9l8ADEbEF7IP1RUR8e8Wc5x5GqzH5wCMRsR/W8yxqUiuB7A+Ip4i2QPgJwA+COBjSGR/nGUdPoK09gUBdEfEKMkygMcBfBrAHwD4ZkTcT/JuAE9HxJd/0eNZ6mfi2wAciIhXI6IC4H4Aty7ymC4YEfEYgPmNrW8FcG/2+F7MvgmXtAbrkZSIOBIRT2WPRwDsB7ARCe2Ps6xDUmLWaPa0nP0LAO8B8HfZ9DdtXyz1JL4RwME5zw8hwZ2eCQCPkPwJyR2LPZgmrI2II9nj1wGsXczBNOlTJJ/JLrcs2csQ85HcAuB6AE8g0f0xbx2AxPYFySLJvQAGAOwG8AqAoYg43fX6TctVSz2Jn0/eFRG/DOBmAJ/M/ouftJi9Frd0r8ed3ZcBvAXAdQCOAPji4g5HQ3IZgG8A+ExEDM99LZX9UWcdktsXETETEdcB2ITZKwZXLtZYlnoS7wewec7zTdm05EREf/ZzAMCDmN3xKTqaXds8fY1zYJHHsyARcTR7I9YA/DUS2B/Z9ddvAPh6RHwzm5zU/qi3Dinui9MiYgjA9wC8E0AfyVL20puWq5Z6En8SwOXZt75tAG4D8NAij+mckezOvsgByW4ANwLYd/bfWrIeAnB79vh2AN9exLEs2OnEl/kQlvj+yL5MuwfA/oj40pyXktkfjdYhwX2xhmRf9rgTszde7MdsMv9wFvam7YslfXcKAGS3G/0ZgCKAXRHxnxZ5SOeM5GWYPfsGgBKAv0lhPUjeB+AGzP6ZzaMA7gTwLQB/C+BizP654I9ExJL+0rDBetyA2f++B4DXAHxizrXlJYfkuwB8H8CzAGrZ5D/G7DXlJPbHWdZhO9LaF9dg9ovLImZPhP82Ij6fvc/vB7ASwE8B/KuImPqFj2epJ3EzM2tsqV9OMTOzs3ASNzNLmJO4mVnCnMTNzBLmJG5mljAncTOzhDmJm5kl7P8Bhck492gvgfsAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iSCJwv5eLEmh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "89eaeaea-d3e9-49b5-b3f6-f1850375374c"
      },
      "source": [
        "features[3].numpy()[0][:,:,0]"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-20.555742  , -12.208584  ,  -7.0306067 ,  -7.6002955 ,\n",
              "        -13.286039  , -16.944511  , -22.576359  , -20.112274  ,\n",
              "        -13.79982   , -15.922814  , -18.935764  , -22.68362   ,\n",
              "        -21.403494  , -16.73529   , -18.61959   , -17.375011  ,\n",
              "        -15.307136  , -18.268656  , -21.505444  , -20.143517  ,\n",
              "        -19.690437  , -21.366829  , -20.582407  , -18.859013  ,\n",
              "        -22.817776  , -23.210333  , -21.720505  , -21.488922  ,\n",
              "        -20.987284  , -23.001572  , -29.822376  , -26.525368  ,\n",
              "        -25.755257  ],\n",
              "       [-20.632622  , -12.673341  ,  -8.989402  ,  -8.937446  ,\n",
              "        -25.334885  , -12.503162  , -13.229435  , -12.7696495 ,\n",
              "        -15.790423  , -18.356163  , -22.0742    , -19.277525  ,\n",
              "        -19.280014  , -17.900768  , -21.656599  , -18.38562   ,\n",
              "        -15.856417  , -20.752113  , -21.278421  , -19.468042  ,\n",
              "        -23.1034    , -23.408281  , -20.408955  , -21.358932  ,\n",
              "        -20.023367  , -20.899197  , -20.382637  , -18.981632  ,\n",
              "        -21.023462  , -23.48347   , -29.069878  , -32.57917   ,\n",
              "        -36.01108   ],\n",
              "       [-14.655233  , -22.26807   , -12.135625  , -10.694906  ,\n",
              "        -10.932352  , -11.574893  , -14.43536   , -11.72495   ,\n",
              "        -11.772084  , -13.37649   , -15.436908  , -23.846504  ,\n",
              "        -20.079126  , -23.135536  , -18.873093  , -16.34559   ,\n",
              "        -15.222767  , -19.997719  , -22.686012  , -21.092813  ,\n",
              "        -23.664019  , -26.962488  , -21.089745  , -20.868284  ,\n",
              "        -18.293407  , -19.11648   , -19.24767   , -19.199741  ,\n",
              "        -26.091667  , -22.830456  , -26.097067  , -29.941395  ,\n",
              "        -33.123714  ],\n",
              "       [-22.252129  , -14.175931  , -16.82492   , -13.9914465 ,\n",
              "        -14.001235  , -11.016535  , -13.661508  , -14.073634  ,\n",
              "        -12.80017   , -12.333311  , -15.884909  , -17.619541  ,\n",
              "        -19.600266  , -19.240492  , -16.402979  , -15.414029  ,\n",
              "        -15.86289   , -18.651188  , -26.707687  , -22.450542  ,\n",
              "        -24.347572  , -24.375374  , -19.418848  , -18.721172  ,\n",
              "        -19.648422  , -22.151379  , -22.821886  , -20.581005  ,\n",
              "        -21.04052   , -24.965113  , -24.873964  , -26.511755  ,\n",
              "        -30.593168  ],\n",
              "       [-12.703052  , -11.176442  , -10.724829  , -10.4230995 ,\n",
              "        -12.279322  , -12.6526985 , -13.588256  , -14.366526  ,\n",
              "        -12.482203  , -13.187906  , -20.32913   , -19.361624  ,\n",
              "        -20.75817   , -19.341389  , -17.060072  , -21.367146  ,\n",
              "        -18.557272  , -20.363842  , -24.308609  , -22.637548  ,\n",
              "        -21.005571  , -19.409927  , -18.937393  , -23.40691   ,\n",
              "        -27.433258  , -22.653126  , -22.672012  , -29.3643    ,\n",
              "        -21.32923   , -22.387203  , -23.609087  , -26.562853  ,\n",
              "        -34.23046   ],\n",
              "       [-21.371586  , -11.5002    ,  -8.298484  ,  -8.144961  ,\n",
              "        -11.883619  , -13.236864  , -15.118477  , -13.057186  ,\n",
              "        -11.045307  , -12.519656  , -15.220755  , -17.908237  ,\n",
              "        -16.895948  , -17.801214  , -36.914284  , -18.437069  ,\n",
              "        -17.589685  , -20.159126  , -23.318186  , -28.107658  ,\n",
              "        -23.247223  , -20.005434  , -20.106335  , -29.089144  ,\n",
              "        -31.388687  , -22.931166  , -23.473843  , -19.932217  ,\n",
              "        -20.196995  , -22.5578    , -23.430243  , -27.619095  ,\n",
              "        -40.106674  ],\n",
              "       [-15.350807  , -15.644933  , -10.085054  ,  -9.109588  ,\n",
              "        -10.871908  , -13.003651  , -23.14283   , -11.217264  ,\n",
              "        -12.09882   , -16.396038  , -14.232246  , -17.460354  ,\n",
              "        -16.603941  , -18.05037   , -20.530247  , -19.854836  ,\n",
              "        -17.21241   , -20.044323  , -22.30583   , -25.224354  ,\n",
              "        -30.754066  , -22.274315  , -19.90356   , -21.890518  ,\n",
              "        -21.660612  , -19.405346  , -18.237694  , -18.959301  ,\n",
              "        -21.627495  , -27.230728  , -24.73246   , -33.89375   ,\n",
              "        -28.66741   ],\n",
              "       [-22.177374  , -21.28085   , -11.900608  , -12.762569  ,\n",
              "        -15.232965  , -18.99453   , -13.273124  , -11.185752  ,\n",
              "        -10.631002  , -13.29738   , -16.647337  , -19.848267  ,\n",
              "        -19.438309  , -23.240437  , -21.11486   , -19.095045  ,\n",
              "        -16.153616  , -19.421875  , -20.447016  , -22.692888  ,\n",
              "        -22.967403  , -19.58612   , -22.853956  , -20.97622   ,\n",
              "        -20.9151    , -20.273687  , -18.819908  , -21.40598   ,\n",
              "        -26.527302  , -21.969316  , -22.478077  , -24.309708  ,\n",
              "        -26.216675  ],\n",
              "       [-20.635658  , -17.66636   , -11.1259575 ,  -8.80482   ,\n",
              "        -10.088063  , -11.75164   , -15.462055  , -12.841885  ,\n",
              "        -11.053935  , -15.211548  , -19.90218   , -16.926004  ,\n",
              "        -20.08031   , -21.367443  , -19.61726   , -16.416258  ,\n",
              "        -14.45567   , -16.699684  , -24.512077  , -23.682468  ,\n",
              "        -21.316511  , -17.968008  , -18.128977  , -18.943548  ,\n",
              "        -23.981316  , -25.496244  , -22.189293  , -24.945509  ,\n",
              "        -22.891752  , -25.387268  , -23.571453  , -21.928396  ,\n",
              "        -21.48507   ],\n",
              "       [-21.371832  , -13.294294  ,  -8.434479  ,  -7.7998114 ,\n",
              "         -9.96549   , -10.834521  , -14.625471  , -12.997695  ,\n",
              "        -13.059904  , -18.414875  , -18.675905  , -18.686293  ,\n",
              "        -19.006948  , -19.452759  , -20.93404   , -18.729778  ,\n",
              "        -14.537748  , -15.94725   , -19.544434  , -18.968714  ,\n",
              "        -18.352148  , -22.148163  , -17.774822  , -17.20942   ,\n",
              "        -20.827906  , -22.9088    , -22.555431  , -22.2812    ,\n",
              "        -22.013313  , -25.13561   , -29.08498   , -23.86974   ,\n",
              "        -23.051655  ],\n",
              "       [-22.774868  , -12.325323  ,  -7.1358223 ,  -8.613293  ,\n",
              "        -14.559245  , -12.950611  , -15.381516  , -14.9025345 ,\n",
              "        -15.352348  , -18.7298    , -21.727207  , -21.123837  ,\n",
              "        -21.539938  , -21.166702  , -19.908682  , -20.102772  ,\n",
              "        -15.587057  , -18.456299  , -21.12547   , -18.093483  ,\n",
              "        -17.396286  , -20.123444  , -20.18958   , -18.13038   ,\n",
              "        -20.805197  , -24.503038  , -24.166645  , -24.128923  ,\n",
              "        -20.00872   , -20.987139  , -24.139473  , -35.58425   ,\n",
              "        -28.467958  ],\n",
              "       [-14.0450115 , -16.75357   ,  -6.0507593 ,  -6.226288  ,\n",
              "        -12.325613  , -12.088173  , -14.239241  , -16.672602  ,\n",
              "        -16.443995  , -16.792187  , -15.395769  , -19.985693  ,\n",
              "        -19.667576  , -20.321047  , -19.320005  , -17.110016  ,\n",
              "        -16.231462  , -19.48866   , -32.831078  , -21.154497  ,\n",
              "        -17.65958   , -21.475986  , -27.922087  , -19.879097  ,\n",
              "        -21.33287   , -26.372787  , -23.39861   , -21.283243  ,\n",
              "        -20.86937   , -21.91842   , -25.110027  , -30.638706  ,\n",
              "        -30.237942  ],\n",
              "       [ -9.354811  ,  -4.648684  ,  -1.5538003 ,  -3.5667303 ,\n",
              "         -7.298999  ,  -6.000902  ,  -8.693544  , -12.96505   ,\n",
              "        -11.272787  , -10.053942  , -12.562409  , -16.947247  ,\n",
              "        -12.221153  , -11.853935  , -12.265244  , -18.012173  ,\n",
              "        -15.895796  , -20.043364  , -22.275585  , -22.55091   ,\n",
              "        -18.990131  , -18.061703  , -20.123987  , -18.801731  ,\n",
              "        -18.674995  , -20.270308  , -21.439884  , -21.401989  ,\n",
              "        -26.613052  , -27.80231   , -26.927607  , -32.75395   ,\n",
              "        -39.547985  ],\n",
              "       [-12.934269  ,   1.0374011 ,   3.5998783 ,   1.9044464 ,\n",
              "         -3.959684  ,  -4.9418173 ,  -6.1667676 ,  -8.283938  ,\n",
              "         -7.6995845 , -12.149824  ,  -9.114716  ,  -7.9651737 ,\n",
              "         -5.959056  ,  -5.690725  ,  -7.6518803 , -10.949696  ,\n",
              "        -13.562962  , -16.05509   , -15.60008   , -19.848236  ,\n",
              "        -21.96475   , -17.3306    , -15.81235   , -20.76237   ,\n",
              "        -17.347492  , -23.994144  , -24.50118   , -21.568989  ,\n",
              "        -23.73011   , -26.168013  , -26.152918  , -31.730274  ,\n",
              "        -36.568707  ],\n",
              "       [ -4.632152  ,   4.545631  ,   7.396298  ,   5.5899816 ,\n",
              "         -0.35573983,  -3.4101787 ,  -3.6515985 ,  -3.4334552 ,\n",
              "         -4.3295183 ,  -7.567737  , -10.930855  ,  -6.8373785 ,\n",
              "         -3.8211932 ,  -5.093861  ,  -8.1429205 , -12.387005  ,\n",
              "        -12.227069  , -12.930694  , -15.169752  , -13.177448  ,\n",
              "        -12.731911  , -11.791458  , -11.081004  , -12.620454  ,\n",
              "        -15.883866  , -20.651478  , -19.595182  , -19.386274  ,\n",
              "        -21.392382  , -23.761982  , -25.03357   , -29.950113  ,\n",
              "        -30.279913  ],\n",
              "       [ -2.5705476 ,   6.266198  ,   9.570579  ,   7.9166036 ,\n",
              "         -0.38904023,  -1.8590182 ,  -2.9804182 ,  -0.77325076,\n",
              "         -1.7829942 ,  -7.5338607 ,  -9.104952  ,  -5.4011984 ,\n",
              "         -4.247407  ,  -8.283699  , -10.780822  , -12.594555  ,\n",
              "        -13.249063  , -18.405617  , -14.050604  , -10.289426  ,\n",
              "         -8.740405  , -10.265675  , -11.245945  , -11.793503  ,\n",
              "        -15.54677   , -24.87222   , -17.962059  , -22.242237  ,\n",
              "        -22.598263  , -29.848099  , -23.551367  , -24.500797  ,\n",
              "        -27.715237  ],\n",
              "       [ -8.907686  ,   6.491182  ,  10.120682  ,   8.688403  ,\n",
              "          2.5332925 ,  -2.5098565 ,  -3.9634113 ,  -1.6301999 ,\n",
              "         -2.3377373 ,  -5.372817  ,  -6.4374304 ,  -8.338999  ,\n",
              "         -6.635829  ,  -5.5197773 ,  -9.281844  , -13.471378  ,\n",
              "         -9.023923  ,  -8.718613  ,  -8.671124  ,  -9.293942  ,\n",
              "         -9.496513  , -11.864281  , -13.099827  , -14.004478  ,\n",
              "        -17.23867   , -15.8038435 , -17.807701  , -22.328785  ,\n",
              "        -23.628511  , -22.411396  , -27.897291  , -23.058525  ,\n",
              "        -22.451612  ],\n",
              "       [ -4.507425  ,   5.8434076 ,   9.254347  ,   7.309776  ,\n",
              "         -0.11652976,   1.0348358 ,  -2.6437044 ,  -2.1088054 ,\n",
              "         -6.2450953 ,  -3.3949199 ,  -7.668146  ,  -4.5842557 ,\n",
              "         -5.5195003 ,  -6.799262  ,  -8.911343  ,  -9.873107  ,\n",
              "         -5.5001707 ,  -5.4952836 , -10.378704  , -13.40177   ,\n",
              "        -17.248407  , -10.559874  , -11.201559  , -15.835592  ,\n",
              "        -14.076495  , -16.370438  , -18.963028  , -18.468452  ,\n",
              "        -19.376314  , -20.27287   , -23.699038  , -23.523544  ,\n",
              "        -27.494774  ],\n",
              "       [ -7.000784  ,   5.57056   ,   7.936837  ,   2.441112  ,\n",
              "          1.7717288 ,   3.6966398 ,   3.9387753 ,   2.332478  ,\n",
              "          0.41627252,  -0.95706356,  -2.1435463 ,  -3.6351202 ,\n",
              "         -7.7969494 , -11.712028  ,  -9.063672  ,  -6.571681  ,\n",
              "         -3.7213788 ,  -5.4793906 , -14.779495  , -11.497345  ,\n",
              "        -11.606331  ,  -9.143127  , -10.969825  , -17.732485  ,\n",
              "        -13.645206  , -13.566675  , -15.66872   , -15.778353  ,\n",
              "        -17.318901  , -21.01923   , -30.28212   , -24.172047  ,\n",
              "        -24.715755  ],\n",
              "       [ -2.0930865 ,   6.33444   ,   8.307544  ,   4.2272797 ,\n",
              "          0.83927655,   4.2389994 ,   4.812589  ,   2.3814423 ,\n",
              "         -4.813818  ,  -0.47920442,  -7.292844  ,  -4.615549  ,\n",
              "         -7.209501  , -11.68928   ,  -9.935385  ,  -9.130487  ,\n",
              "         -3.7519572 ,  -4.176893  ,  -7.6245975 ,  -9.584898  ,\n",
              "        -12.332745  ,  -8.134084  ,  -9.33441   , -14.444416  ,\n",
              "        -12.400189  , -14.527437  , -21.037786  , -16.600853  ,\n",
              "        -20.097902  , -23.300173  , -24.096956  , -30.558308  ,\n",
              "        -24.379684  ],\n",
              "       [  0.60399115,   7.038686  ,   9.104303  ,   6.4549513 ,\n",
              "         -0.450109  ,   2.9993062 ,  -1.5523345 ,   1.9270566 ,\n",
              "          0.8083593 ,  -2.2321408 ,  -5.980459  ,  -6.0483093 ,\n",
              "         -6.6702127 ,  -8.379779  ,  -7.6755385 , -11.959678  ,\n",
              "         -4.9237328 ,  -5.0964184 ,  -9.380426  ,  -9.418793  ,\n",
              "         -9.943757  ,  -9.735625  , -10.091534  , -11.094027  ,\n",
              "        -13.8323765 , -18.473045  , -17.722412  , -20.069298  ,\n",
              "        -21.593493  , -22.815474  , -22.980349  , -24.98435   ,\n",
              "        -25.25151   ],\n",
              "       [ -9.260559  ,   6.6653175 ,   7.727235  ,   6.649478  ,\n",
              "          2.7609105 ,  -0.09280889,  -0.07929086,   2.6375895 ,\n",
              "          1.0258471 ,  -7.427399  , -10.237896  ,  -6.842499  ,\n",
              "         -7.567616  ,  -9.02026   ,  -6.152748  ,  -6.3904023 ,\n",
              "         -5.4129195 ,  -8.27437   , -15.519451  , -12.244391  ,\n",
              "        -14.627842  , -14.004144  , -16.655659  , -14.250003  ,\n",
              "        -14.140251  , -17.789797  , -20.391104  , -23.776205  ,\n",
              "        -31.198023  , -24.480206  , -24.906391  , -25.63964   ,\n",
              "        -35.72135   ]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333
        },
        "id": "qq0vZvMjt9Qd",
        "outputId": "e93d0f33-21e6-4b69-ba6a-26d39ef6a6fb"
      },
      "source": [
        "from sklearn.datasets import load_sample_image\n",
        "from sklearn.feature_extraction import image\n",
        "\n",
        "import matplotlib as mpl\n",
        "\n",
        "import matplotlib.patches as patches\n",
        "\n",
        "mpl.rcParams['figure.dpi']= 100\n",
        "\n",
        "# Use the array data from the first image in this dataset:\n",
        "one_image = features[3].numpy()[0][:,:,0]\n",
        "print('Image shape: {}'.format(one_image.shape))\n",
        "\n",
        "patches = image.extract_patches_2d(one_image, (22,3))\n",
        "print('Patches shape: {}'.format(patches.shape))\n",
        "\n",
        "for i in range(11):\n",
        "  ax = plt.subplot(1, 11, i + 1)\n",
        "  #ax.figsize=(3, 22)\n",
        "  ax.imshow(patches[i]).set_clim(cmin, cmax)\n",
        "  #im = ax.matshow(features[3].numpy()[0][:,:,0])\n",
        "  #patch = mpl.patches.Rectangle((0+i*3,0),3, 22, transform=ax.transData)\n",
        "  #im.set_clip_path(patch)\n",
        "  plt.axis(\"off\")\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Image shape: (22, 33)\n",
            "Patches shape: (31, 22, 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeUAAAEaCAYAAAAv9E+uAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAPF0lEQVR4nO3dTY9c6VkG4Kquru6q7nZ3+9sej8eezGAPY2VmIAqRWADZRAiJRST+Q4TEBhRQJJRVhBREFDYgxIYVPwAJiR0LFkEsgpIQ0ExChpAhNhl7pj/s/qiuTxZsLDF9qIe8L+dJdF1bPz7366pTddfZPO4uFosOANC+lbYPAAD8N6UMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJKGUASCJ1WUHH/zeH4dWfx29Ng4d5MqNp6H5f/y1P+ie92d3/uIPQ2eNZk9nsd8y3/r1L5171k/8ze+HzhrNPni0HZr/wed+99yzvvnXXwydNZrd3++F5r/3hd85/x748z8KnTWavfnDc6M/0rf+5LfP/QuvfvmrobNGs6dbofHOv3z5/LO++Vux74Fo9vGdWWj+33/z8x951gdfiJ0zmts7jn0O3/38+ffq3T/7Suis0ez+Yex+eedL57//r3wldq9Gswd7sQ2T3/zT81/X174Yuwei2bP12L/t2189/3V9nidlAEhCKQNAEkoZAJJQygCQhFIGgCSUMgAkoZQBIAmlDABJKGUASEIpA0ASS6/ZDJvFVpAdna7/dGQH1c6+/rXg767Pnf9Ho3G/avb7vzQNzbeZfXAxdo81mVypnL0SWx/YmP3x4HsUzF49LPOVdPTKpGru6lG59z+6NjOavX4YGm81+6zg56rN7B+HJ2UASEIpA0ASShkAklDKAJCEUgaAJJQyACShlAEgCaUMAEkoZQBIQikDQBJKGQCSWHrh68mL89CFd/8ptif54OfK7eetnb2yPgvNNxkdxnZfR7Mff2Ycmm8yHsX2A4ezgzvL28zunsbusUb92Gcrmr24UG6neHcc+x0fzZ5eie2sPlc/9pmO5k4vlHueqZ092S531tHN2PsZzd59u9x3wLO7wc4KZo+u1NmV7UkZAJJQygCQhFIGgCSUMgAkoZQBIAmlDABJKGUASEIpA0ASShkAklDKAJCEUgaAJJZeZrz9vVh/H74W29Hcf1xul3Dt7Ol2LzSfNTtqcbhW9frd7UJ7jzudTudp7HUNZ6+W29Xe7QWvFczuHhe8ZypnL9Zi+4pbyy34/ncmwR3K0fdgXnBHc+Xsozuh8Vaze6ex+WV5UgaAJJQyACShlAEgCaUMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJKGUASCJpXdfH3w8thu4O4jtn57tFtwlG9xjHM1eKfhTZnZ7VDV7drL0W/y/Z18c180+LbejebEe3GUcze6Wu18XlbO7i3K7jxe1s0sdNfr2RHNnBfdJV84ePCl41srZ83JfV9Wze7Gvw6V5UgaAJJQyACShlAEgCaUMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJKGUASCJpbd9br/dD1346Zux3cOb27Ed0E0W89jO05LZURtbZ1WvfzQqt096uBE7azh7WvA34kpw+XEwe2VU8KzB+zWaPd8I7gFvMTu6M/88i7Xgju5g7qLg56p29umNcu9/9HMVzV49Kve5qp09uVDw/2t4jidlAEhCKQNAEkoZAJJQygCQhFIGgCSUMgAkoZQBIAmlDABJKGUASEIpA0ASShkAklh69/Xx7dge0d4Ha6H50XqZnbf/H9m91XJnHZ3GzhrNXlkrd9bJJLZzN5o9n5T7jVg7e/c7sX3VjYKXimbv/+I0FtBi9mJc5h7oDuvm9veW/upsPXtacPd57exFdGd9g3k/dq1wdsGvgOd5UgaAJJQyACShlAEgCaUMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJKGUASCJpReZXvh+rL/nwdWwTy+V2yW7+TC2lDSaPZ32Q/ON1zqtm907iu2rbnK2P6iaPdwr9xux+3i9avbeW+X2SQ//I3YPRLNLrujdeyu2UzyavTIot6u9Zu5kp+Ce9srZ/YNy3wGTndhZo9n943J36+px8HUKZi/svgaAn25KGQCSUMoAkIRSBoAklDIAJKGUASAJpQwASShlAEhCKQNAEkoZAJJQygCQxNJLd6extcedyfYiNN9dnccCGpxdqps92BmH5tvMHl8ot1N8ePm0avbpxXJn7dwcVc3u7ZU76+krsfc0mj3rljtrL7hPOJy9UWan+CK4Uz6a2zsp9zyzmMeWKEez1w7KLWmunT3eiX13t5m9dlhn+bUnZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJKGUASAJpQwASShlAEhCKQNAEkoZAJJYekHsSnAl7dnVWWh+88JZLKDF7N5KuT3dG5WzJ+Nye4/7q7HXNZq90o9dv0l0K200e7Fa7nVtMztqUTl6cdLOvy2a2z8suPu6cvZ8LTTeavbqabl90rWz55VuVU/KAJCEUgaAJJQyACShlAEgCaUMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJLH09s7e2SJ45dj8cG0Su36L2Sdn/dB8m9mzJ4PQfJPpLPYbLpq9WC+3U7x2dq/cmu7O/Dj2nkazV07L/fbuVs7uFroFaueWOmen0+n0juuede1pbL7N7EUvNt+YfVg5u+A98DxPygCQhFIGgCSUMgAkoZQBIAmlDABJKGUASEIpA0ASShkAklDKAJCEUgaAJJZes3nwemynWP+DpS/d6XQ6nfXb09B8m9lrq+XOuh68VjT7ZGMjNN9kd/O0bnY3uMq1wWIjuA8ymD3vd2PXb7Ia+2xFs1eDaxybs2OvUzi70MvaGwUvFByfbpS7V1cmsfBo9ni33FlrZw+elLtXj16Kfa6i2YtKj7SelAEgCaUMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJKGUASAJpQwASShlAEhi6SXR/WuxvccvP/gwNP+xC7H5xmt96r2q2Y9HW6H5Jm9cflQ1e3Qjtge8yc3Np1Wzh/1yO8Uv3Tismr1/PAzNN7n14l7V7JMnm6H5JsNbR1Wzu4PgzvJzTC7GrhPN7a2VOWen0+lMb4yrZs+O+qH5Jou7sR6IZo/H5XbKz7Zjr1M0e3JtEppflidlAEhCKQNAEkoZAJJQygCQhFIGgCSUMgAkoZQBIAmlDABJKGUASEIpA0ASShkAklh6OfF4fxC68Kdf/25o/uX1x6H5xuyrdbOfbG6H5pv8ys7bVbOn89dD800+ffk7VbNPp+V29N7eju2+jmYvyq2T7mz2Y7uPo9krK4vYX2iwuxnbfRzNXl2Zh+bPs3U9tqM7mnvwpNz++52Lx1Wz13dHofkm05PYPvto9tq1cvvva2ffGJZ7XZ/nSRkAklDKAJCEUgaAJJQyACShlAEgCaUMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAkll5kev/ew9CF3xy+F5q/3/8wNN9m9qPV/dB8k0+ux17XaPZ725dD803eGvygavb3j8ud9f72+1WzN/tnofkmO+uxfdLR7JubT0PzTa5vPKuaPZqV2X9+/0psn32p3P+LWzuxPe1Rh4cbxa61ezW2Uzyavb1Zbp/0eBR7T6PZlwYnoflleVIGgCSUMgAkoZQBIAmlDABJKGUASEIpA0ASShkAklDKAJCEUgaAJJQyACShlAEgiaV3X18bxnbe3ujFdt6+uDoMzbeZ3e+W23naZnbUndW62Tv9cntva2cfT9d+YrJvDsrtVh70plWzj2brofnz3NnYq5r76Ol2aL7JrY2Dqtnrw0lovkm3u6iavbk2Ds23mT1d1Hmm9aQMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJKGUASAJpQwASShlAEhCKQNAEkvvvv76w5dCF165GduR2u/2QvON2cH9rNHszW653zK1s+8NfhSabzP770b3QvNtZp8U3H19No/dA9Hs03m5s9bOHs+X/kpqNffixmlovs3s9X5sX3mb2U+ebYXmm+xuxXb1R7NPB2eh+WV5UgaAJJQyACShlAEgCaUMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJKGUASCJpZeu3r28F7rwaBHc5zsfh+abtpTWzt6bz0Pzl1vMfnd0LTTfZvaDrf8MzbeZ/e1nt0LzbWafzcrsk+50Op2tfmzfbzT7uwdXQ/Nt5d7aOgzNt5m9d1zmNe10Op3Lg+Oq2S9d3A/NNxmN+1WzT6ex6y/LkzIAJKGUASAJpQwASShlAEhCKQNAEkoZAJJQygCQhFIGgCSUMgAkoZQBIAmlDABJLL0g9slx07bp/2kS3D+9Nz8JzTedpnb2357cC83/TGN2bPd1NPvmWrkdvbWzX+iX23tbO/v+oNye7lfX36+a/Xi6HZpv8htXv141+6VhbMf+eT61829Vc/enG6H5Jp+5+U7V7AvXY/vKm7wwjH2uotn742FovsnPX/9h1eyHhzuh+WV5UgaAJJQyACShlAEgCaUMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAklDIAJKGUASCJpXdfv3HlUejCm91JaH6t2w3Nt5l9Y/UgNN9m9j+f3g7Nt5k9WI/tAW/ybDaomt3vzELzjdkr7WVHbXTL7VH+KH//wceqXr9U7hsXH/7EZH/zyQuh+Sb9buzei2Zf3TwOzbeZvdqr8zn0pAwASShlAEhCKQNAEkoZAJJQygCQhFIGgCSUMgAkoZQBIAmlDABJKGUASEIpA0ASS+++fnnjg9CFN1amsfluLzTfZvbd/l5ovs3srx3dC823mf3hdCs03+RwNqya/Q/PXg3N/2qL2Q82y+1p/qv9T1TN/uWr/xqaP89kEftcRXPfPbkamm8ze9iPfR822RtvVs0+Gq+H5tvM7vfmoflleVIGgCSUMgAkoZQBIAmlDABJKGUASEIpA0ASShkAklDKAJCEUgaAJJQyACShlAEgiaV3X//lO58MXfizv/CN0PzePLandLvhzyaL2G+NaHavU25Pd+3snx0+Cs03Zy+qZr9z+kJovs3sS/3j0Hyb2YPuJDTfZvZRt8w+4dq533h8KzTfpFc5++lRbA98m9mXdsp9rn6019QSP372ZFquB57nSRkAklDKAJCEUgaAJJQyACShlAEgCaUMAEkoZQBIQikDQBJKGQCSUMoAkIRSBoAkuotFbJ8xAFCHJ2UASEIpA0ASShkAklDKAJCEUgaAJJQyACShlAEgCaUMAEkoZQBI4r8AjnLfOPqWOUAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 600x400 with 11 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 409
        },
        "id": "pJTD1RL0mw2V",
        "outputId": "e150f155-35f1-401c-b011-82eacfcf47d1"
      },
      "source": [
        "plt.matshow(features[3].numpy()[0][:,:,:])"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7fbc428eb410>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAFmCAYAAAAbPSIgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUCUlEQVR4nO3dfaxkZ10H8O+v1G6x7G616K64EmvB+oKkoURaY9pKJWKigSoKotEa/EMs8S0GbYLhLbFKcKmBFQOplj9Q/KNYQ2JbUhTfUlesAQR5p4Bl+yKs3F0M3UJ5/GPm6vQ6d3fv3Jk5c/f5fJKT6Zxznnt++/TczPc+c855qrUWAKAvZw1dAACwfAIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0aEcFgKq6rqo+VVUPVdXhqvreoWsaWlW9oqrahuXDQ9e1bFV1RVW9o6qOjPvguRu2V1W9qqruq6ovVdWdVfXkoepdptPom5unnEO3D1XvMlTV9VX1nqo6XlUPVtWtVXXxhn3OrapDVfX5qvpiVd1SVfuGqnlZTrNv3j3lnPmjoWpehqp6cVW9v6qOjZe7quqHJ7bvuPNlxwSAqnp+koNJXpnkaUnel+SOqvrGQQtbDR9M8k0Ty/cPW84gzsvonLhuk+0vTfLLSX4xyTOS/HdG58+5yylvUKfqmyS5PY8+h35qCXUN6cokh5JcluRZSb4myTur6ryJfV6X5EeT/MR4/yckefuS6xzC6fRNkrw5jz5nXrrMIgdwb5LfSnJpkqcn+eskf1lV3z3evvPOl9bajliSHE7yhon3ZyX5bJLfGrq2gfvlFUneO3Qdq7QkaUmeO/G+ktyX5Dcm1u1N8lCSFwxd75B9M153c5Jbh65t4H75hnHfXDFxfjyc5HkT+3zHeJ/Lhq53yL4Zr3t3khuHrm3oJcnRJC/aqefLjhgBqKpzMkpdd66va619dfz+8qHqWiFPHg/vfrKq3lpVTxy6oBVzYZL9efT5s5ZRqHT+jFw1Hu79SFW9saouGLqgJds7fj06fr00o798J8+ZDyf5TPo7Zzb2zbqfrqrPVdUHquqGqvraZRc2lKp6TFW9IKPRtbuyQ8+Xs4cu4DQ9PsljkjywYf0DGaWsnh1Ocm2Sj2Q0DPfyJH9fVU9prR0fsrAVsn/8Ou382R9uz2io8p4kFyX5nSS3VdXlrbVHBq1sCarqrCQ3JvnH1toHxqv3J3m4tfaFDbt3dc5s0jdJ8qdJPp3kSJKnJvm9JBcn+bGlF7lEVfU9GX3gn5vki0muaa39e1Vdkh14vuyUAMAmWmu3Tbx9f1UdzugX8yeT3DRMVewkrbW3Tbz9t6p6f5JPJLkqybsGKWq5DiV5Svq8duZUpvZNa+1NE2//raruS/KuqrqotfaJZRa4ZB9JcklGoyLPS/KWqrpy2JJmtyO+AkjyuSSPJNl4ReW+JPcvv5zVNU6gH03ypKFrWSHr54jz5zS01j6Z0e/cGX8OVdUbkvxIkh9ord07sen+JOdU1fkbmnRzzpykb6Y5PH49o8+Z1trDrbWPt9bubq1dn9HFtb+SHXq+7IgA0Fp7OMndSa5eXzcemro6o+EYxqrqcRkN4943dC0r5J6Mfgknz589Gd0N4PzZoKoOJLkgZ/A5NL4t9A1JrknyzNbaPRt2uTvJl/Poc+biJE/MGX7OnEbfTHPJ+PWMPWc2cVaSXdmh58tO+grgYEbDLf+S5J+T/GpGF2D8yaBVDayqXpvkHRkN+z8ho9skH0nyZ0PWtWzj4DP518eF4+/ljrbWPlNVNyZ5WVV9LKNA8OqMvr+8dfnVLtfJ+ma8vDzJLRmFpIuSvCbJx5PcseRSl+lQkhcmeU6S41W1/j3tWmvtS621taq6KcnBqjqa5FiS1ye5q7X2T8OUvDQn7Zuqumi8/a+SfD6jawBel+TvWmvvH6LgZaiqG5LcltGFfbsz6oOrkvzQjj1fhr4NYYu3XLwkow+6ExkNOT1j6JqGXpK8LaMPshMZ3af6tiQXDV3XAP1wVUa33Gxcbh5vrySvyuhD7qGMrtb99qHrHrpvkjw2ow/6BzO6jelTSd6UZN/QdS+4T6b1R0ty7cQ+52b0YXg0o+dGvD3J/qFrH7pvknxLkr/N6MP/oSQfyyg07hm69gX3y03j348T49+XO5M8ayefLzUuHADoyI64BgAAmC8BAAA6JAAAQIcEAADokAAAAB0SAACgQzsqAFTVrqp6RVXtGrqWVaNvptMvm9M30+mXzemb6XZqv+yo5wCMH9+6lmRva+3Y0PWsEn0znX7ZnL6ZTr9sTt9Mt1P7ZUeNAAAA8yEAAECHVm4yoKqqjCa1OT5l8+7119FuTNA30+mXzemb6fTL5vTNdKvWL7uTHGmn+I5/5a4BqKpvzmhSGwBgNgdaa5892Q6r+BXAtL/8AYDTd8rP0lUMAADAgi0sAFTVdVX1qap6qKoOV9X3LupYAMDWLCQAVNXzkxxM8sokT0vyviR3VNU3LuJ4AMDWLOQiwKo6nOQ9rbWXjN+fleQ/kry+tfa7p2i7/kAFAGA2p3wo0dxHAKrqnCSXJrlzfV1r7avj95dP2X9XVe1ZX/J/t1MAAAuyiK8AHp/kMUke2LD+gST7p+x/fUZ/8a8vbgEEgAVbhbsAbkiyd2I5MGw5AHDmW8STAD+X5JEk+zas35fk/o07t9ZOJDmx/n5FnqIEAGe0uY8AtNYeTnJ3kqvX140vArw6yV3zPh4AsHWLmgvgYJK3VNW/JPnnJL+a5Lwkf7Kg4wEAW7CQANBa+/Oq+oYkr8rowr/3Jnl2a23jhYEAwABWcTIgzwEAgO1Z/nMAAIDVJwAAQIcEAADokAAAAB0SAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0SAAAgA4JAADQIQEAADokAABAhwQAAOiQAAAAHRIAAKBDAgAAdEgAAIAOCQAA0CEBAAA6JAAAQIcEAADokAAAAB0SAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0SAAAgA4JAADQIQEAADokAABAhwQAAOiQAAAAHRIAAKBDAgAAdEgAAIAOCQAA0CEBAAA6JAAAQIcEAADokAAAAB0SAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdmnsAqKpXVFXbsHx43scBAGZ39oJ+7geT/ODE+68s6DgAwAwWFQC+0lq7f0E/GwDYpkVdA/DkqjpSVZ+sqrdW1RM327GqdlXVnvUlye4F1QQAjC0iABxOcm2SZyd5cZILk/x9VW32wX59krWJ5d4F1AQATKjW2mIPUHV+kk8n+fXW2k1Ttu9Ksmti1e4IAQCwHXtba8dOtsOirgH4X621L1TVR5M8aZPtJ5KcWH9fVYsuCQC6t/DnAFTV45JclOS+RR8LADg9i3gOwGur6sqq+taq+r4kf5HkkSR/Nu9jAQCzWcRXAAcy+rC/IMl/JvmHJJe11v5zAccCAGaw8IsAt2p8K+Da0HUAwA52yosAzQUAAB0SAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0SAAAgA4JAADQIQEAADokAABAhwQAAOiQAAAAHRIAAKBDAgAAdEgAAIAOCQAA0CEBAAA6JAAAQIcEAADokAAAAB0SAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0SAAAgA4JAADQIQEAADokAABAhwQAAOiQAAAAHRIAAKBDAgAAdEgAAIAOCQAA0CEBAAA6JAAAQIcEAADokAAAAB0SAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0aMsBoKquqKp3VNWRqmpV9dwN26uqXlVV91XVl6rqzqp68vxKBgC2a5YRgPOSvC/JdZtsf2mSX07yi0mekeS/k9xRVefOVCEAMHfVWpu9cVVLck1r7dbx+0pyJMnvt9ZeO163N8kDSa5trb3tNH7mniRrMxcFAOxtrR072Q7zvgbgwiT7k9y5vqK1tpbkcJLLpzWoql1VtWd9SbJ7zjUBABvMOwDsH78+sGH9AxPbNro+o7/415d751wTALDBKtwFcEOSvRPLgWHLAYAz39lz/nn3j1/3JblvYv2+JO+d1qC1diLJifX3o8sIAIBFmvcIwD0ZhYCr11eMv9d/RpK75nwsAGBGWx4BqKrHJXnSxKoLq+qSJEdba5+pqhuTvKyqPpZRIHh1RncG3DqPggGA7ZvlK4CnJ/mbifcHx69vSXJtktdk9KyANyU5P8k/JHl2a+2h2csEAOZpW88BWATPAQCAbVv6cwAAgB1AAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0SAAAgA4JAADQIQEAADokAABAhwQAAOjQ2UMXMF9vnrHdL8zYrmZsBwDDMgIAAB0SAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0SAAAgA4JAADQoWqtDV3Do1TVniRrs7RtmfHfMmsXzDgZ4OxzCJp9EIDTsre1duxkOxgBAIAOCQAA0CEBAAA6JAAAQIcEAADokAAAAB0SAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6NDZQxewuYNJHru1Jsue2HDJswgCwLwYAQCADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0SAAAgA4JAADQIQEAADokAABAhwQAAOhQtbbsKfROrqr2JFlL1pLs2VLbFfunbGr2yQC3MY3gOTO2+/LshwRgMHtba8dOtoMRAADokAAAAB0SAACgQ1sOAFV1RVW9o6qOVFWrqudu2H7zeP3kcvv8SgYAtmuWEYDzkrwvyXUn2ef2JN80sfzUDMcBABbk7K02aK3dluS2JKna9Kr0E621+7dRFwCwQIu6BuCqqnqwqj5SVW+sqgs227GqdlXVnvUlye4F1QQAjC0iANye5GeTXJ3kN5NcmeS2qnrMJvtfn9FN/+vLvQuoCQCYsK0HAVVVS3JNa+3Wk+zzbUk+keQHW2vvmrJ9V5JdE6t2J7nXg4Dm29KDgAC6MvyDgFprn0zyuSRP2mT7idbasfUlyfFF1wQAvVt4AKiqA0kuSHLfoo8FAJyeLd8FUFWPy6P/mr+wqi5JcnS8vDzJLUnuT3JRktck+XiSO7ZdLQAwF1sOAEmenuRvJt4fHL++JcmLkzw1yc8lOT/JkSTvTPLbrbUT26gTAJgjswEO4a3/NVOz+umvn3Mhp3PQ5R8SgG0b/iJAAGD1CAAA0CEBAAA6JAAAQIcEAADokAAAAB0SAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdWuHpgP84ydduqW1rz19ITfNWM06x29rsc/PO3PKDM7Z7ys/M2PDsGdt93YztXjdjO4CVZjpgAOD/EwAAoEMCAAB0SAAAgA4JAADQIQEAADokAABAhwQAAOiQAAAAHRIAAKBDAgAAdEgAAIAOCQAA0KFZp15bvLUfT/bsGbqKhZh9AsbZZ26ceSLBGRtel2fO1O4P8+qZ2iX3zNgOoE9GAACgQwIAAHRIAACADgkAANAhAQAAOiQAAECHBAAA6JAAAAAdEgAAoEMCAAB0SAAAgA4JAADQIQEAADpUbfap6RaiqvYkWcva2pZnA1ytf8nmPjRju+/c1lFnnQ5wxqPVsv9vHJqx3UvmWgXAitjbWjt2sh2MAABAhwQAAOiQAAAAHRIAAKBDAgAAdEgAAIAOCQAA0CEBAAA6JAAAQIcEAADokAAAAB0SAACgQwIAAHTo7KEL2Mwf7N2bx2610YrNbLiZ75qx3Xb+dbPOBTjrMf91xoZPm3nSwjfO2hCgS0YAAKBDAgAAdGhLAaCqrq+q91TV8ap6sKpuraqLN+xzblUdqqrPV9UXq+qWqto337IBgO3Y6gjAlUkOJbksybOSfE2Sd1bVeRP7vC7Jjyb5ifH+T0jy9u2XCgDMy5YuAmytPXvyfVVdm+TBJJcm+buq2pvkRUle2Fr76/E+P5/kQ1V1WWvtn+ZSNQCwLdu9BmDv+PXo+PXSjEYF7lzfobX24SSfSXL5tB9QVbuqas/6kmT3NmsCAE5h5gBQVWcluTHJP7bWPjBevT/Jw621L2zY/YHxtmmuT7I2sdw7a00AwOnZzgjAoSRPSfKCbdZwQ0YjCevLgW3+PADgFGZ6EFBVvSHJjyS5orU2+Rf7/UnOqarzN4wC7Btv+39aayeSnJj42bOUBABswVZvA6zxh/81SZ7ZWrtnwy53J/lykqsn2lyc5IlJ7tpmrQDAnGx1BOBQkhcmeU6S41W1/r3+WmvtS621taq6KcnBqjqa5FiS1ye5yx0AALA6thoAXjx+ffeG9T+f5Obxf/9akq8muSXJriR3JPml2coDABZhq88BOOUX9K21h5JcN14AgBVUbcVm0Bs/C2AttZbUni21bY8spqbNLfeCxRpgPsD20IyHO3e2Wme/BtTFowAT9rbWjp1sB5MBAUCHBAAA6JAAAAAdEgAAoEMCAAB0SAAAgA4JAADQIQEAADokAABAhwQAAOiQAAAAHRIAAKBDW50OeHlOPofBVMe23mSHWf4/8NiskwE9fMb/zwDY0VZxNsBvTnLv0HUAwA52oLX22ZPtsIoBoJI8IcnxKZt3ZxQODmyyvWf6Zjr9sjl9M51+2Zy+mW7V+mV3kiPtFB/wK/cVwLjgqaml/m+y+OOnmue4N/pmOv2yOX0znX7ZnL6ZbgX75bRqcBEgAHRIAACADu20AHAiySvHrzyavplOv2xO30ynXzanb6bbkf2ychcBAgCLt9NGAACAORAAAKBDAgAAdEgAAIAOCQAA0CEBAAA6JAAAQIcEAADo0P8AZvXjFiBDoK0AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 600x400 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0XKelyTlfrv"
      },
      "source": [
        "### (Rearrange3d)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aBj2kguDkHo_",
        "outputId": "45c7deb1-6fe5-492f-c547-ff57094a0066"
      },
      "source": [
        "features[4].numpy()[0].shape"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(22, 33, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 413
        },
        "id": "Z9BUCE2blkW3",
        "outputId": "61395476-f1ee-446b-b56f-38b323d3a77b"
      },
      "source": [
        "plt.plot(features[4].numpy()[0][0,:])"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fbc42862fd0>,\n",
              " <matplotlib.lines.Line2D at 0x7fbc4286e210>,\n",
              " <matplotlib.lines.Line2D at 0x7fbc4286e3d0>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhAAAAFZCAYAAADJvxawAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5yU1fX48c+dmZ3tvbEssEvvIL2qqKiIir1FTYIlX02iMcZIrFGjv5jExG6MhmAjllhRUEQFFaTj0jssu8CyvfeZeX5/3HlmZ9nZMtuB83699rWz85S5g8icvfecc5VhGAghhBBC+MPS1QMQQgghxIlHAgghhBBC+E0CCCGEEEL4TQIIIYQQQvhNAgghhBBC+E0CCCGEEEL4TQIIIYQQQvjN1tUDaG9KKQX0BEq7eixCCCHECSgcOGo00yjqpAsg0MHD4a4ehBBCCHEC6wUcaeqEkzGAKAXIzMwkIiKiq8cihBBCnDBKSkro3bs3tGAW/2QMIACIiIiQAEIIIYToIJJEKYQQQgi/SQAhhBBCCL9JACGEEEIIv0kAIYQQQgi/SQAhhBBCCL9JACGEEEIIv0kAIYQQQgi/SQAhhBBCCL9JACGEEEIIv0kA0c0YhkEz+5cIIYQQXU4CiG6krKaMOR/PYe7SuVTUVnT1cIQQQohGSQDRjSw7tIz0knQ2Zm/kvu/vw2W4unpIQgghhE8SQHQjnx741PP4m8xveGbjM104GiGEEKJxEkB0E1llWaw/th6A34z9DQALti/ggz0fdOWwhBBCCJ8kgOgmFh9cDMD4xPHcMvIWbh99OwCPr3mctVlru3JoQgghRAMSQHQDhmHw2f7PALi4/8UA3D76di7oewEOw8FvV/yWg8UHu3KIQgghRD0SQHQDuwp2sb94P3aLnXjLeNYdLADgT9P+xOj40ZTWlPKrr39FUVVRF49UCCGE0CSA6AbM5MmJiaczd/42rv7Xai549ns+Tcvlb2c8TXJYMpmlmdy14i5qnDVdPFohhBBCAogu53A5+Pzg5wAkWafhcOkmUruOlXLP/zZzybObmRZ2L6G2UDZmb+TR1Y9KoykhhBBdTgKILrY2ay15lXlEBUaRl5MKwM+npjJv1hASIwLJKa1m/vJKSjKuQ2Fh0f5FzN82v2sHLYQQ4pQnAUQXM5cvZqXOYn16CQAzhyZy+4z+fH/v2Tx9zWiGJUVQUTyAymM6wfLZTc/y4toPZCZCCCFEl5EAogtV1FbwTcY3AEyIO5es4ipsFsXYlCgA7DYLl43pxeI7p/PfWydxRuIl1BRMBeCfOx5n1j8X8knaEWqd0rFSCCFE55IAogt9nfE1lY5KUiJSKCzsAcCoXpGE2G31zlNKMbV/HPN/PoFFP/krCbbTUBYHhwNf4q73V3DmX5fz3obMrngLQgghTlESQHShzw7o3g8X9ruQdQcLAZjUL7bJawYnRrLo6pfpFzEAi62U8JTXOVpazL3vb2FfTmmHj1kIIYQACSC6TG5FLmuy1gBwUd+LWJeeD8CkvjHNXhsaEMrL575EbFAshj2LxP7vAy5PECKEEEJ0NAkgusiSg0twGS5Oiz8NqyuOzIJKrBbF+NTmAwiApLAknj/7eQKtgVTYthKYsJiNhySAEEII0TkkgOgi5vLFxf0vZu1BPfswomcEYYG2pi6rZ2T8SJ6Y/gQA9thVrD+ys/0HKoQQQvggAUQX2Fu4l10Fu7BZbJyXch5rD+jW1c3lP/hyfur5TEicBEBWzRYKyqVTpRBCiI4nAUQXMGcfzkg+g6igKNa6975oSf6DL9OSpwBgDd3HjxmyjCGEEKLjSQDRyVyGi8UH9NbdF/e/mJySKg7mlaMULc5/ON7kpMkA2EIOsOFQXruNVQghhGiMBBCdbP2x9WRXZBNuD+eMXmewxj37MCwpgsjggFbdc0jMEIIsoShrNasy09pzuEIIIYRPEkB0MnP54vzU87Fb7aw9YJZvuvMfDEN/+cFqsTI6fgIA+0rTcEhnSiGEEB1MAohOVOmoZNmhZQBc1O8igLr8h34xUFUCz4+D+edBabZf9z4nRbe4NoL2sOuYNJQSQgjRsSSA6ETfZn5LeW05yWHJjEkYQ15ZNftyygCYmBoD2z+Cgv1weB3Mnwl5e1t878k9dR6ENTiDtenHOmT8QgghhEkCiE5k7rx5Yb8LsSgL69yzD0N6hBMdaoct7+oTrXYoyoD550LG2hbdOzUilRBLDMri4NtD6zpk/EIIIYRJAohOkl+Zz6ojqwCv5YsDXu2rC9Ph0CpAwS1fQ/J4qCyEN+bAzs+avb9SipGx4wHYWbSpQ96DEEIIYZIAopN8kf4FTsPJiNgR9I3sC3jnP8TClvf0if3OhKRR8LNPYdAF4KiC926Eda82+xrn9p0OQLllF7ml1R3zRoQQQggkgOg0n+3XswgX9dezD4XlNZ5kx4mp0bD5bX3i6Ov0d3sIXPMWjJsLhguW3ANfPdJkhcaMPjqR0hJ0hJUHMjrmjQghhBBIANEpDhYfZFv+NqzKyqzUWQCsS9ezDwMSwogr2gIFByAgFIZcVHeh1QYXPQ1nP6h/Xvk0fHQbOHy3q04MTSTM0hOlDJYd+KFD35MQQohTmwQQncDs/TAteRqxwbrfg2f/i74xdbMPw+ZAYFj9i5WCM34Pl7wEygpb3oH/XqVLPn0YGjUWgK0FGzrgnQghhBCaBBAdzLt1tZk8CXh24JySEg7bPtBPjr628RuNuR6uf0/PUhxYAQtmQ0lWg9POSZ0GQKFrBzUOaSglhBCiY0gA0cHSctI4UnaE0IBQZvSeAUBxZS07svQMwumuDVBVDBHJkHp60zcbMBPmLobQBMjeqss8c3fXO+WigaeDoVD2HFam7++ItySEEEJIANHRzN4PM/vMJNgWDMCG9AIMA/rGhRK51z37MOpqsFibv2HPMXDLMogdAMWZumtlxhrP4cigSEJIAWDJ3u/b980IIYQQbhJAdKAaZw1L05cCeudNk1m+eXZvBXu/1E+OamL54njRqXDTl9BrAlQVwetzYMciz+FBEWMASMtd37Y3IIQQQjRCAogO9N3h7yitKSUxJJEJPSZ4njcbSM2xrQGXA5JOg4Qh/t08NBZ+uggGXwjOavjfz+HYVgDOdJdz5ji2Yfi5MZcQQgjREh0aQCilYpRSC5VSJUqpIqXUfKVUWDPnP6+U2q2UqlRKZSilnlNKRXbkODvKp/v18sXsfrOxKP1HXVbtYNtRnf8wNGeJPtHs/eAvewhc8yYMng2GE758EAyDy4ZOw3BZMaxFrD+yu/n7CCGEEH7q6BmIhcBw4FzgIuAM4JUmzu/p/roHGAH8HJgFzO/QUXaA4upivjvyHQAX96tbvtiQXoDTZTA9Kg97dhpYbDDiita/kMUKs/6s9884sAL2fU1saDhBrv4ALNr9bVvehhBCCOFThwUQSqmh6A//WwzDWGsYxkrgDuBapVRPX9cYhrHNMIwrDMP41DCM/YZhfAM8AFyslLI18jqBSqkI8wsI76C35JcVmStwuBwMih7EwOiBnufN/Ie5Ye4NrwacC2HxbXux6FSY+Av9eNlD4HLSL/Q0ADbkyMZaQggh2l9HzkBMAYoMw/DuaPQV4AIm+XGfSKDEMAxHI8fvA4q9vg63YqztbmP2RgCmJ0+v9/zaA/koXEwpW6afaKr3gz9O/x0ERUHODkhbyPReenvvrOptOF3O9nkNIYQQwq0jA4geQI73E+4goMB9rFlKqTjgIZpe9vgzOsgwv3q1ZrDtzQwgxiWO8zxXUeNgy+FiJlt2ElKVDUGRMGhW+7xgSAycea9+/M0TzOk7EsMZiEtVsCV3R/u8hhBCCOHmdwChlHpSKWU08+VnSYHP14kAFgM7gEcaO88wjGrDMErML6C0ra/dVrkVuWSUZqBQjEkY43l+06EiHC6DG4Lc+1QMvxwCgtrvhSfcopczyo6RsmcB1poBAHy2R/IghBBCtK/WzED8HRjazNcB4BiQ4H2hO48hxn2sUUqpcOALdDBwmWEYta0YZ5fZmKNnHwbHDCbcXpeSsfZgPsFUMdNwN35qbfVFY2yBMPMRANQPzzEoUAcQa7LWtu/rCCGEOOX5TExsimEYuUBuc+cppVYDUUqpcYZhbHQ/fTY6aGn0E80987AUqAbmGIZR5e8Yu9rGYw2XL0BvoHW+ZQOBRiVE94XeE9v/xYddqhtMHV7PLbV7uScQDlfuoMZZg91qb//XE0IIcUrqsBwIwzB2omcRXlVKTVRKTQNeAN4xDOMogFIqWSm1Syk10f1zBPAlEArcDEQopXq4v1rQ57l72JSzCYCxCWM9z1XVOknLLOJyq7u99Ojr9E6b7U0pOO8JAM7N+hKLIxgXNaTlpLX/awkhhDhldXQfiOuBXcDXwBJgJfALr+MBwGAgxP3zWHSFxkhgH5Dl9dW7g8faLoqri9lbuBeAsYl1AcSPGUVEO/OYbt2mnxh1dccNos8kGDoHi+FibKWuwFh2cGXHvZ4QQohTjt9LGP4wDKMA+EkTx9MB5fXzCu+fT0RpOWkYGKRGpBIXHOd5fu3BfC61rsKCAX2mQkzfjh3IzEdg9xLmVGWxITyWVUfWNHeFEEII0WKyF0Y781W+CbB2f77X8sU1HT+Q2P4w4VYmVekUksMVuymrKev41xVCCHFKkACinfkKIKodTiozf2Sw5TAua6BOdOwMZ95LvAqhd20tBi7P2IQQQoi2kgCiHVXUVrAjXzdt8s5/2HK4mIvR+2KoIbMhOKpzBhQSQ8Xk3zKpUs9CfJ8h/SCEEEK0Dwkg2tGWvC04DAc9QnvQM7Ruu4/1+7KZY10FgGrv3g/NiDjjlwyuCgRgzYGlnfraQgghTl4SQLSjTdl15ZvKq0SzYtcy4lUJlfYY6H92p45JBQSTF349AIdcJeTl7erU1xdCCHFykgCiHfnKf6h1uhieuwSAysGXgTWg08cVNOwn9KnWj9d9+2inv74QQoiTjwQQ7aTWWcuW3C1A/QBix4EMzkZvSBo1+addMraxqbGUlQ0D3G2tc3d3yTiEEEKcPCSAaCfb87dT5awiOjCafpH9PM8Xrv8fgaqWIwF9sfQc3SVjG94zguxKvb33uqBAWPbHLhmHEEKIk4cEEO3EXL4Ym1g//yHp0McAHEm5pGNaV7dAoM3KkOhRYFg4EmAj88CXcPA7v++zPGM5Ny65UcpBhRBCSADRXnztf+HI3c/g6m04DUXExEYbcnaKCSlJOCr7ALAmOAi+fBBcrhZf/+6ud7lrxV2k5abxyb5POmqYQgghThASQLQDp8vJj9k/AjCuR13+Q97qNwFYo0YxcMCgLhmbaWyfaJzl/QFYGxIOWZth6/+avc4wDJ7b9ByPr30cl6EDjpzKnA4dqxBCiO5PAoh2sK9oH6W1pYTYQhgcPVg/aRiE7NAf0NvjZmO1dO0WH2NTonGWDwBgbVg4LoCvH4PaykavqXXV8uCqB3l166sATEueBkBuRbO7uQshhDjJSQDRDjZk6yqLMQljsFnc+5Md3kBE1WHKjUBswy/qwtFpiRFBJAYOwnAFUOSqZm90Lyg5DJvf9nl+RW0Fd3x9B4v2L8KqrDwy5RHuHnc3IAGEEEIICSDahaeBlFf7alfOTgDWu4YwbkCvLhnX8calxOOs0LuArhkwVT+Z9t8G5+VV5jF36VxWHV1FkDWIZ896lisGXUFiSCIAhdWF1DhrOm3cQgghuh8JINrIMAyfDaRycnWeQJklnOE9I7pkbMcb1ycKh7mMYXWBssLh9ZC313NORkkGNy65kR35O4gOjGb++fM5s/eZAETYI7Bb7ADkVsoshBBCnMokgGijQyWHyK/Kx26xMyJuhOf53Dz9ARsUHo3N2j3+mL3zIDbkb6W2/zn6gHsZY2vuVm78/EYOlx0mOSyZNy54g1HxozzXK6WID4kHZBlDCCFOdd3jk+0EZpZvjogbQaA10PN8ZUkBAIFh0V0yLl+GJkVgd/XE5Qil0lHJ1gE6KZLN7/Bd5gpu/vJmCqoKGBozlLdmv0VqZGqDeySEJACQUyGVGEIIcSqTAKKNfC1fANSWFwIQEhHT6WNqTIDVwqheMTgrdKfMtXYrBEXxkauIO7/5DZWOSqb1nMaCWQuIC47zeY/4YPcMhCxhCCHEKU0CiDZqLIAwqkoAiIr2/UHcVXQ/CL2MsSZ7Ay/3G8PD8bE4cTGn/xyeP+d5QgNCG71eZiCEEEKABBBtcqz8GEfKjmBRFk5LOM3zfI3Dha1WBxCxsfFdNTyfxqVE43A3lNqUs4kXK/cDcGtJOY+Pu5cAS9O7hUoOhBBCCJAAok3M2YchMUPq/daeUVBOBBUARHazGYgxfaIwamNx1UYBoFDcX2Xjzvx81M5FzV7vmYGQbpRCCHFKkwCiDcz+D8cvX+zPLSfcHUCooMhOH1dT4sICSYkNpaZgChEBMfxjxj+4btiN+qCPnhDHSwiWJQwhhBASQLRJY/kPB3LLiVDl+oduFkAAjOsTTW3BmVwe9wozU2bCqGsABRk/QMGBJq+VJQwhhBAgAUSrFVYVsr9Y5w9478AJcDCnhHDce0x0wwBiTIouLU3LLNJPRCZD/7P0483vNHmtuYRRVltGRW1Fh41RCCFE9yYBRCuZ/R/6R/YnOqh+r4ejuXlYlKF/COoeXSi9je2j8x/SMopwutzjHO3ebnzz201u8x0aEEqILQSQUk4hhDiVSQDRSr72vzDl5+n8AJfFDragTh1XSwxODCfUbqW02sHenFL95JALITACijL0UkYTpJRTCCGEBBCt1Fj+Q2F5DS53DwgVFAmqa7fx9sVmtXCaexZi5d48/aQ9BIZfqh83k0wpeRBCCCEkgGiF8tpydhbo3TYbJFDmlXlKOFU3XL4wnTesBwCfbj5a96S5jLH9Y6gua/Racwai2y1hbHoDPp8HLmdXj0QIIU56EkC0wuaczbgMF8lhyfQI7VHv2P5uXoFhmj0yCYuCzYeLSc9zj7fPZIjuC7XlsPPTRq81SzmzK7I7Y6gts/V9WHQHrH0Z9i/v6tEIIcRJTwKIVtiQvQFoOPsAuoSzO1dgmOLDA5k2QDe58sxCKAWnmcmUjS9jdLsljMMb4eNf1v18aFXXjUUIIU4REkC0glmBcXz5JsCB3DKvGYjuu4QBcPHongAs2nwUwzCrMa7V3w9+rxMqfTADiG6RRFl8BN65DpzVuEL1zAiHmk4CFUII0XYSQPip2lnN1tytQCMzEHl1bay78wwEwPnDe2C3WtibU8auY+5qjKg+kHo6YMDmd31eZy5hdHkORE25Dh7KsiFhGPOC/giA68hGqK3s2rEJIcRJTgIIP23L20aNq4aYoBhSIlLqHXM4XRzKLydcuQOIwO49AxEZHMBZQ/RswidpXsmU3ssY5syEF+8lDMPH8U7hcsHHt0PWZgiJpezyt3j/aBTZRhQWVy3OzPVdMy4hhDhFSADhJ+/9L9RxJZqZhZXUOg2iLWYORFRnD89vc0YnAzoPwhMMDJ0DAaG6rXXm2gbXxAfrAKLKWUVpbWmnjbWeb/8COz4BSwBcs5BtFVEYhmKdawgA2374omvGJYQQpwgJIPzUWP8H0PkPAEmB1fqJbr6EAXDO0ARC7VaOFFWyKaNQPxkYBsMu0Y999IQIsgURGajfW5ckUm77EL59Uj++6GlImcK2I8UApFmGA1Cx7zsKyms6f2xCCHGKkADCDw6Xg7TcNKDxCgyAuAAzgOjeSxgAQQFWzhuuS1EX1VvGuE5/3/6Rz3wCcxai00s5j2zSSxcAU34NY/VOolsO6wCi79hzARht7OGpz7d17tiEEOIUIgGEH3YX7qa8tpzwgHAGRg1scPxAnp6BiLacGEmUpjnuaozFW7NwON37YKRMh8g+UF0CuxY3uMbTTKozZyBKsuCdn4CjCgaeB+c+5jm01T0D0XvIWByBUYSoanZs/J4fzVkVIYQQ7UoCCD+Y+Q+nJZyG1WJtcHy/ewYi1DixAojpA+OIDgkgr6yG1Qfy9ZMWS11Jp49lDHMGotMqMWordfBQmgXxQ+CK+eD+b1BSVctBdzOskb2isaVOA2CiZScPf7K9bsMwIYQQ7UYCCD80lf8AdUsYQU53G+huXoVhCrBamD0yCTi+GsO9jHFgOZQcrXdNp26oZRjwya/g6CYIjoHr3qm3PGTmPyRHBRMTaoeUqQBMDdjN1iPFvL3Odz8LIYQQrScBRAsZhlGvAuN4JVW15JXp3Adbrd5M60SZgYC6ZYyl245RVeveSyKmH/SZAoYLttTvCdGp3Si/+xts+wAsNrjmTYjpW+/wVnf+w6he7j9vM4Cw7cWCi78t3S0JlUII0c46NIBQSsUopRYqpUqUUkVKqflKqbBmrvmXUmq/UqpSKZWrlPpEKTWkI8fZEgeLD1JYXUigNZDhscMbHDdnH3qFKZTT/WF1AiRRmiakxpAUGURptYMVu72CgtHuWYi0t+v1hDCbSeVUdvAMxI5PYPkT+vGFf4fU6Q1OMfMfRpoBRI9RYA/D7ihlVnwBxZW1/PWLXR07TiGEOMV09AzEQmA4cC5wEXAG8Eoz12wE5gJDgfMBBXyplGqYdNCJzP0vRsWPIsAa0OC4WcI5PNb8kFVgD++s4bWZxaK4aJRexqi3Q+fwS8EWDHm7dQWEW6csYWRthg//Tz+edDuM+7nP0zwBRLI7gLDaoPckAP4wTOd0vLM+s65MVQghRJt1WAChlBoKzAJuMQxjrWEYK4E7gGuVUj0bu84wjFcMw/jOMIx0wzA2AQ8CvYHUjhprS5j7XzSX/zAkyh1ABEXoRMQTiNlU6qud2ZRVO/STQZEw9CL92GuDLXMJI68iD5fhav/BlB6Dt68DRyX0PwfOe9znacUVtRzK10mrngACPMsYfUrTuHJcLwAe/mSbJFQKIUQ76chPuClAkWEYG7ye+wpwAZNacgOlVCh6NuIgkNnIOYFKqQjzC2j3X/sNw2DDMf02fG2gBXUlnP3C3fkDgSdO/oNpRHIEfeNCqXa4WLbjWN0Bs7X11vfBofM8YoNjUSgchoPCqnb+zd7lhHdvgJIjEDcIrlqgZxV82HZUzz70iQkhKsRedyBFV2Jw6Af+MGsw4UE2th0pkYRKIYRoJx0ZQPQA6s1vG4bhAArcxxqllPqlUqoMKAMuAM41DKOxLLj7gGKvr8NtHHcDR8uPkl2RjU3ZGB0/2uc55gxEaqjXb+4nGKWUJ5myXjVG3zMhvCdUFcHuzwEIsAQQExQDdEAp5+7P4fB6XcVy3TtN/lmaDaTqzT4AJI8FayCU5xJXnck95w0G4G9Ld5PvTnYVQgjRen4HEEqpJ5VSRjNfbU16XAiMAc4E9gDvKaWCGjn3z0Ck11evNr52A2b1xbDYYYQEhDQ47nQZnj4EPYPNBMoTL4AAmHOaDiBW7s2rq1ywWGH0Nfrx5rc953ZYHsSal/T3CbdAbP8mT916pAjwSqA02QKh13j9+NAqrp/Uh6FJEe6Eyt3tO14hhDgFtWYG4u/oBMemvg4Ax4AE7wuVUjYgxn2sUYZhFBuGsdcwjO+AK4EhwGWNnFttGEaJ+QW0++5OZv+HsYm+ly+OFlVS7XBht1qItVbpJ0+gCgxv/ePDGN4zAofLYMnWrLoDo93LGHuXQZkOGDqklPNoGhxapUs2J97a7OlmAuWo42cgwJMHwaHV2KwW/nSJrp55d4MkVHYVp8vg5W/3882uTm6BLoRod34HEIZh5BqGsauZrxpgNRCllPLOOjzb/ZoNt3hsnHJ/Bfo71vYyp/8cbh15K2f3Odvn8f3uCozUuBAs1foD7USdgYC6nhCLvKsx4gdB8ngwnJ5ZCLMbZbuWcpqzD8Mvh4hGc20BKCyvIbNA79MxvMkA4gcAxqfGSEJlF3tjdTpPfr6LXy38keLK2q4ejhCiDTosB8IwjJ3AF8CrSqmJSqlpwAvAO4ZhHAVQSiUrpXYppSa6f+6nlLpPKTVOKdVHKTUV+B9QCSzpqLE2Z2ziWO4ceydjEsb4PG7mP/SLC9N7R8AJ04XSl4vcAcT69AKyir020hr3M/199YtQW0liSCLQjksYJVm6YRTA5NubPd2cfUiNDSEyuGFpLb0mgrJCcQYU6eTJP1wwRBIqu0h2SRV//3IPAJW1Tv63wWdetBDiBNHRdYbXA7uAr9EBwErgF17HA4DBgJlYUAWc7j53H/AuekliqmEYndAzuXU8FRjxoVB14s9AJEcFMyE1GsOAzzZ7LWOMulZvsFWWDRv+0/5LGOtfBZdDd79M9r1c5K2ugVSU7xMCw6DnafrxodUAxIUFSkJlF3nssx2UVTsIC9QVNa+vTpdZICFOYB0aQBiGUWAYxk8Mwwg3DCPSMIybDMMo8zqebhiGMgxjhfvno4ZhzDYMI9EwDLthGL0Nw7jeMIxunfXmmYGIDzspAgiAOafpnhCfbD5S96TNDmfcox+vfJqEAD3L0i4zEDUVsGGBfjz5ly26xNPC2tfyhcmzjLHK89T1k/owTBIqO9WK3Tks3pKFRcHrN00kKiSAzIJKvtnVbX8vEEI048TqdNRN1QUQoVBl7oNx4i5hAMwe0QOrRbHtSImnyyage0JEpUB5LvEHVwLtVMa55V2oLND3HnJhiy4xZyBGNBlA1PWDMNmsFv50qSRUdpaqWicPf7IdgLnT+jIuJZprJ/QB4LUfDnbl0IQQbSABRBuVVzs4VqIrL/rHnTwzELFhgUwfEAccl0xpDYAz7wUgftNCAPIr83G4HK1/McOANf/Ujyfd5tmmuyn5ZdUcKdL5GSOSmwjW+kwGFOTv9VSPAIxLkYTKzvLi8n1kFFTQIyKI3547CIAbp6RgUbBqXz57stu9cEoI0QkkgGgjs/9DbKidyJCAkyaAgPrVGIbXRlqMuhai+xJTlocVhYFBfmV+619o39d6rw17OIy5oUWXmLMP/eJDCQ/ykUBpCo6GRPfmZ16zEFA/ofKjH4/4uFi01b6cMl7+dj8Af7x4mM5/qKkgOSKQ84frfnILVqV34QiFEK0lAUQbmSWc/eJD9RMnQRWG6bzhiQTaLBzILWf70ZK6A1YbnHkvFiDOqVt3t2kZY82L+vvYG1u89N8YSysAACAASURBVLO1sQ6UvhxXzmmKCwvk9hm6UdWCVQfrB0mizQzD4KGPt1HrNDhrcDyzRvSA3N3w9DB4fiy/HqiXjj768TBFFbLduhAnGgkg2qheCSecVDMQ4UEBnDNU9wKrt0MnwMirIaY/ibW6lj+7opWNgXJ2wv5vQFlg0v+1+LIGO3A2pZEAAuC6CX0ItFnYfrREciHa2cdpR1h9IJ9Am4XHLhmBMgxYdCdUFkLhQYZ9cTWPRS2mptbBu+ulpFOIE40EEG10IM8rgdLpgBp3wmFQI6WFJxjvZQyXd56A1QZnziPenIEobuUHgNk4asiFEJ3a4ss8HSgbK+H01scdQGRv0x9eXqJD7Z73+PoPh1r8+qJpxRW1PP7ZTgDuPGcgvWNCYNNrkLkGAkJh6ByU4eSnVQt5x/4nvly1HoezA3Z1FUJ0GAkg2uiAZwnDq4kUnPBVGKYZgxMID7SRVVzFhkPH/YY+8kriA/TMS87+L/2/eXkebH5XP578qxZflltaTVZxFUrB8J4t+HMOT4SY/oABGQ2boP5saioAS7ZmkeNOiBVt85elu8gvr2FAQhi3nt5PNwlb9kd98JyH4eo34LJ/YdjDmGjZzYLqu9i+9D9dO2ghhF8kgGgDl8s4roTTvXwREKKrFU4CQQFWznMnuy3afFyiocVKQuoMAHKzNta9/5basACc1dBzjLtaomW2uWcf+seHERroe5vvBsxljIyGyxgjkiMZlxKNw2Xw9jqZSm+rTRmFni6fj186ArvNAp/fqwPs5HF6jxOlYPS1qNtWciRsJBGqktHrfgcf/qKuFFoI0a1JANEGx0qqqKx1YrMo+sSEnFQJlN7MHTqXbD1G7XHTzPGpZwKQixPWvNzymzqqdedJ0I2jlGrxpVta0kDqeD76QXj76ZQUABauPdTgPYqWczhdPPDRNgwDrhjbi8n9YmHXYti5SG+QdvFz9ct0Y/piuflznnVegdNQuh/Iy9N9zhQJIboXCSDawJx96BMbQoDVclIlUHqb1j+W2FA7BeU1rNqXV+9YQoiencixWfUeGZVFLbvptg91S+zwJBh2qV/jMbfwbrKB1PHMGYijP0JNeYPDF4xIIj48kJzSar7Y1uRmsaIJr/2Qzs6sEiKDA7h/9hA9m7DY3b106h3QY0SDa5Kiw9kz9NdcXfMwBQFJUHQIFlwAK57UeUVCiG5JAog28OyBcRJWYHizWS1cOCoJOK6pFF5begfYobq4LimyKYZRV7o58VbdItsPdQmUfvw5R/WBiF56r43D6xscttss/GSi7o74xup0v8bT3by9LoPb39pIYXnnlkZmFVfy9DK9WdZ9FwwhNiwQvn4MSo9CdF84c16j186dmspGYzDnVj5B9dAr9a6vK/4Mr82GwvROegdCCH9IANEG5gxEf7MHxEnSxtoXs1Jh6bZjVNU6Pc8nhOgyzyIF1QrdUbKioOmbpa+EY1vBFgzj5vo1juySKrJLqrEoGNaSBEqTUk2WcwL8ZFIfbBbF+vRCth/1M5+jm3C6DP68ZCefbzvGg59s69DXyizN5HcrfsfzPz7P/qL9PPbpDsprnIztE8XV43tD5jpY/2998sXPQEBwo/calxLNiOQI8h1BzE+8Dy5/VS8FZq6Fl0+HLe916HsRQvhPAog2aNBE6iSdgQAY2yea5Khgymuc9TZAirBHEGgNBCA3cZjOA1n9YtM3M2cpTrsOQmL8GofZQGpAQhgh9hYmUJqaCSASI4J0syPgzdUnZknnlsNFlFTpaf/FW7L4bMvRZq5oncKqQm5bdhtfHvqSV7a8wqWfXMq35X8gMG4Fv5kVh8VVq3s+YMBp10O/GU3eTynFz6f2BfSfvWP4lXDbSug9Wf+d+vBW+PhXevZKCNEtSADRBvV24YS6JMqTMICwWBQXuZcxvtpZ1zRKKUV8sHsZY+z1+sm1Lzc+C5G/H3Z/rh9Put3vcdQ1kGpFnw0zkfLwep3E6cPP3SWdH6cdOSG7I36/V+eoBAfoRMWHPt5Gbmn7blle5ajijm/uIKM0g56hPTk9+UwwrFiDjmGP/4Jff38lN7w/i4XVmeSFxcF5j7fovheNSiI21E5WcRVf7siG6BT4+WI46wFQVkh7S7c9F0J0CxJAtFJVrZOjxXozp35xx81AnGRVGKbJ/WMB+DGjfqKkuYyREz8QeozSzbR+eN73Tdb+CzBgwLkQP8jvMbQq/8EUNxBC4sBRpZMpfRiXEs2wpAiqal28t+HEK+n8fq9uKT5v1mCGJUVQWFHL/R9tbbc23U6Xkz98/wc2524mwh7BP2f+k1THrynd8wCBRdcwPmEiCsXm6lyejI3hnPhQbl05j4/2fkRJTdPlmUEBVq6f5N6l09wfw9023dOl9Ifn2uV9CCHaTgKIVjqYV45hQGRwADGh7iTAk3gJA2Bs72hAv/f8srrfaj2JlJW5MOM+/eTaf+lGUd4qi+DHt/TjKb/0+/UNw/CUcPpVgWGqlwexqpFTFD+bqks631xz6ITapbO0qpZN7uDunKGJ/OOa0QRYFct2ZLfbZmFPbXiKrzO+JsASwLNnPYuzOp5XvjsArhCemHkLC2b9m69dPZmXX8gognBhsCZrDQ//8DAz3p3Bnd/cyRcHv6DK4bth1/WTU7BZFOvSCzz9PgC9S6uywsFvIWtLu7yXU8W/t/6bWR/M4qtDX3X1UMRJRgKIVvJuIKXMHgaeAOLknIGIDAlgYIJertnkNQthLmHkVObA4Asg6TSoLW/42+KmN/TzCcOg31l+v352STV5ZdVYLYphSa38M26mHwTAnNHJRAYHkFlQyYrdOY2e192sOVCA02WQGhtC75gQhvSI4K6Zepbnj4u2c6y4bV0239zxJm/t1AHgE9OfYFziOB74eBsOl8HMoQmcNywRfnyT+EOruaHSycI5/2PJ5Uu4c8ydDIgaQK2rluWZy/n9d7/nl1/7DiATI4KYPVIvlb32Q3rdgegUGHaJftxcjo3w2HBsA89uepYjZUe4e8XdLNi2QDaNE+1GAohW8rSwNks4wSuAODn2wfBlXIqehdjo1dbas4RRkaN/yz/rfn1g3atQ5t6l0+mAda/ox5Nv96txlGnLYR20DEwII9hubebsRng6Uq5ttMdAsN3KtRN6A8d9iHVz5vLF6QPjPc/93xn9GN07itIqB/M+2NLqD49lh5bxt/V/A+DucXczK3UWT3+1l3UHCwgOsPLInOGo8lz48kF9wVn3Q3QqvcN7c+uoW/noko/4cM6H3DziZkB/sJXXNuzHAfDzaakALEo7Sp7XTBdT79Dft70PxbL9enMqHZU8/MPDAPQK64WBwT82/oNHVz9Krau2i0cnTgYSQLRSvU20TCf5EgbAWHcAsclHAJFb4Q4WBp6nWxbXVsCqZ/RzOxdBcSaExOqdPFvBrx04G5M4XOeo1JRC9tZGT7thcgpK6aREs9qms7kMF7//9vfc+c2d1DibT+g0EyhPHxjnec5mtfD3q0Zht1n4dk9uq3a9TMtJ477v78PA4JrB1/CzYT/j71/u4bmv9wJw/+wh9IoOgc/n6f8Hkkb7TJAdGD2Qu8bdRY/QHhgY7Mjf4fP1xvSOYnSvSGqcLt5xt8QGIHmsnkFyOWDdv/x+H6ea5zY9R2ZpJokhibx38XvMmzAPheKDvR9w+1e3N5uTIkRzJIBoJXMGor93AHGStrL2Zs5AbD5cRI1Dt3yuNwMBenbBzIVYPx9Ks+tKN8ffDAFBrXrtNiVQmizWun03Dq1u9LTeMSGcM0S/r64q6dySu4Uv0r9geeZyXt7cdJvwzIIKDuaVY7UopriTXU0DEsL5/XmDAfjTZzvILKho8RjSi9O545s7qHZWM6PXDOZNmMdfvtjNC8v3AfDghUO5cUoq7FkK2z/UeQoXP6eTHxsxMm4kANvyfPepUEp5ZiHeXHNca3FzFmLDa1Bd2uL3carZlL2JhTsXAvDI1EcIt4dzw7AbeO7s5wi2BbM2ay03LLmBzNITL1FYdB8SQLSCYRgNSzjhlJiB6BcXSlRIANUOFzuydMDkKeOszK07ccBM6DUBHJXwwc26dNJqhwm3tOp1DcPw9IAY2cgW3kVVRaQXpzd/s2YSKU0/nZIKwAcbD1NW3fktlZemL/U8nr9tPltzG58xMWcfxvaJIjyo4UZuN03vy/iUaMprnNz7/pb6W7M3Ir8yn9u/up2i6iJGxI7gydOf5P8t2cO/vjsAwKNzhnPL6f2gugwW/05fNOWX0PO0Ju87PHY4AFvzGn8/s0cmERcWSHbJca3FB54PsQN119NNbzb7Hk5F5tKFgcFlAy5jevJ0z7EZvWfwxgVvkBCSwMHig1y/+HrSctK6cLTiRCYBRCvkllZTWu3AoiAlNkQ/aRhenShP3gBCKcXYPvWXMcwqjPLa8rp1be9ZiPTv9fcRV+qttVvhaHEV+eU12CyKIT3CfZ7zy69/yZyP5/BNxjdN38w7kbKJnIDpA+LoFx9KabWDjzYdbtW4W8tluPjykN4iPTUiFZfh4oFVDzRavWDmP0wfEO/zuNWieOqq0QQHWFl9IJ831zQ9q1LpqOTOb+7kcNlhksOSee6s5/nzkgMscJdXPnHZCM826Cx/Qi9PRfWp+2/eBHMGYnve9kbPCbRZuWGyu6TTOw/FYqmr4FnzT9krw4cXfnyBQyWHSAhJ4J4J9zQ4PiRmCG9f+DZDY4ZSWF3IzUtvZsmBJV0wUnGikwCiFfa7Zx96x4QQaHMn89WU6/79cNJWYZg8iZQZOoAIDQglNEAv5XjyIAD6n607CZpaUbppMmcfBiWGExTQMIGyrKaMrXlbMTC4f+X9HCg+0PjNkk7TbbQrCyB3d6OnWSyKn07WJZ2vrz7Uqdnrm3M3k1ORQ1hAGP85/z/EB8dzsPggL/z4QoNznS7Ds8nZ6YPiGhw3pcaF8ocLhgDw5Oe7SM/zncTodDmZ9908tuRtITIwkpfOfom/f3GUt9ZkoBT89YpRXD9J/7lwZKNuHAZw0dNgD/V5T2/DYoehUBwtP0p+ZX6j5/1kUh8CrIqNhwo9CbQAjL5O59IUZ8DOT5p9vVNJWk4ab+7QMzN/nPJHIuy+/y1KCEngtVmvcVbvs6hx1TDv+3m8vPllqdAQfpEAohXqNtHykUBpsUFASBeMqvMcPwMBjSxjKAXnPKzXxQfNgh4jW/2a5g6cjeU/7C6sCwTKa8u5a/ldlNU0kvxos0PvCfpxM8sYV4zrRajdyr6cMlbvb/zDrr2Zyxdn9T6L+JB4Hpn6CABv7HiDTdmb6p1rtq+OCLI1u8X5jZNTmNIvlspaJ/f8b3ODPheGYfCX9X9heeZy7BY7z8x4lheXlfLO+kwsCv5x9Wiudleo4KyFRb8Bw6UTYwfMbNF7C7OH0TdSt63ent/4LERCeBAXjdJ7sHgaS4HeU2PCrfrxD8+3b3trlxMObwTXibele5WjiodWPYSBwSX9L+GMXmc0eX5IQAhPz3ianw37GQAvpr3I/Svvb1HCrhAgAUSr+Mx/8E6gbEWJ4olkdO9IrBZFVnEVR4t0N04zkTK7Irv+yanT4K6tcNVrbXrN5hpI7SrYBcBp8ad51ncfWPkALqORD4IW9IMACA8K4PKxvQB4fXW63+NuDZfh4st0vXxxfur5AJzR6wwuHXApBgYPrnqQitq6REgz/2HagDhs1qb/l7ZYFH+9chRhgTY2HCrkPysP1jv+xo43eHvX2wD8adrjvLnCwgebDmO1KJ65dgyXjelVd/L2j3QlS3A0zPqzX+9xRJze1rupPAioay3+6Zaj5JR6Ld9MuAVsQbqjaDP/DVvMMOCTX8G/z4bvn2qfe3aiF9NeJL0knYTgBH4/4fctusZqsXLPhHt4eMrDWJWVzw58xq1f3kphVWHzF4tTngQQrXDg+E204JRIoDSF2G2eRk5mP4gGpZzeIpOb3ImxOYZheLoSNjYDYZYETuk5hadnPE2AJYBvMr/h31v/7fum3htrNfMbrNmZctmObA4XtryCobV+zPmR3MpcwgPCmdJziuf5eyfcS2JIIpmlmTyz6RnP8776PzSld0wID144FIC/fbmbfTm6mmFp+lKe2qA/OH879m4+W53AJ2lHsVkUL1w3xrMjq0fmOv39tOshtPGlE1/MAKKxSgzT6N5RjOkTRa3T4O21XhUDYfEw+lr9eHXDZZ1WSfsvbH677p5VJ06ZY1pOGq9vfx2AP079I5GB/v07dNWgq3hp5kuEBYSxKWcT1y+5noPFB5u/UJzSJIBoBU8PCJ9NpE7+AAIaNpQyEyk9pZzt6HBhJYUVtQRYFYMbSaA0ZyCGxAxhVPwoHpj0AKATyr4//H3DC5LHgyUASo9CYXqTrz8gIZxpA2JxGbBwbUaT57YHz/JFn7OwW+2e58Pt4Tw29TEA3t71Nmuz1tZrX+3d/6E510zozYzB8dQ4XPzuvc2sPrKG+7/XDcCuHnQtP2wawZKtxwiwKv55wzgucHeHrCdrs/6e1HTVhS/epZzNrbubsxBvrT3kKR0GYMqv9ffdSyBvr99jqCd3NyxxJxxaA/X/zxsXtO2encR76WJO/znNLl00ZmrPqbw1+y2Sw5LJLM3khiU38Jd1f+GTfZ+wp3CPNJ8SDUgA4adqh9NTR1+vB4SnAuPkTqA0eRpKuRMpE4LdMxCVPmYg2sjs/zC4R3hd0qqXamc1B4p00uSw2GEAXDHoCq4cdCUGBvO+n0dGyXEf/PYQ3ZgIWjQFbpZ0vrMug6paZyvfSfOcLifLDi0D6pYvvE1NnspVg64C4OFVD7N8TwZOl0HfuFB6x7Q890YpxZOXjyIiyMb2khXc9tXt1LhqOCN5Bgd2z+SrnTnYbRZeuXE85w7zUTnjckK2e/YgaZTf73NQ9CBsFhtF1UUcKWu6q+QFI5KIDw8kt7Sar712giVuIAy6QD9uS3vr2kr431zd+KzvmXDhU3X3rG1b++/O8FLaS6SXpBMfHM+9E+5t0736R/Vn4eyFjIofRUlNCW/tfIsHVz3IFYuuYPLCyVz72bU8uvpR3tv9HltytzRaFSRODRJA+CkjvwKXAWGBNuLDA+sOVLmzxE+xGYjtR0uoqHHUbajlawmjjZrbwntf4T4choOowCgSQ+o+7O6beB+j4kdRWlPKb5b/pl7eAFB/GaMZM4cmkhwVTGFFLZ9tyWrdG2mBTTmbyKvMI8IewZSkKT7P+d3435EclszR8qO8sk3vN+LP7IMpMSKQMyf9SHDyu7hwMD7+DIoOXcXyXXkE2izM/9l4znI302ogf7/+wA0IgdgBfr+23WpnSLSuCGluGcNus3DVOJ178fbxnTTNxlKb3264eVtLfXEf5GyH0Hi4/FUYdS1E9IKybEhb2Lp7dpLNuZt5fYdeunh4ysN+L134Ehscy3/O/w9Pnv4kNwy9gXGJ4wgLCKPGVcP2/O28v+d9/rTmT1y/5Hom/3cyl31yGfd9fx+vb3+d9cfWy0zFKUQCCD/t97WJFpxySxg9I4PoERGE06V3yGzQjbIdmSWcjeU/7CzYCejlC+//JnarnadnPE1sUCz7ivbp5jre0+WeRMqmKzFA91G4wSzp/CG9w8rdzOWLc/qcQ4C1YUMo0GWzf5r2JwAO1X6DNXQ30wf4F0DUOGu4b+V9rMjRH5DVeWfy3coLWLW3lOAAKwvmTmg6p+KYe0fMxBG6u2crDI/TDaWaCyBAL7mAzveo10kzZSr0HKO3aF/fSL5LU7Z/5F6qUHD5K7pPic1eF5iserbb9pqodlbz0KqHcBkuLup3ETN6z2i3ewdaA7mw34XMmziP12a9xqrrVrH4ssU8deZT3DLyFqb1nEZMUAxOw8m+on18duAzntrwFDctvYkbl9xIcXVx8y8iTngSQPjJZwkneFVhnBoBhFKqXh6Edxlne364GobR7B4YZv7D0NihDY4lhCTwjxn/wKZsLE1f6kk0A6D3REBB4UEoaX5W4ZoJvbHbLGw9UsyPmUXNnu8vh8vR5PKFtwk9JjCnr95TJCjpA0b0tjd5vrfCqkJu/fJWFh9YjE3ZuHvMA4SWz8HpUoTarbx+00Sm9m8mIMlydy9sxfKFycyDaK4SAyAlNpSp/WMxDPjfBq9ZCKXqciHWvaKXI1qq4CAsulM/nv5b3bfENPanutdE0SEdZHRDL6W9xMHig8QFx/GHiX/o0NeyKAt9Ivpwfur5/Gbsb3j53JdZcfUKvrryK144+wV+ddqvOLv32YQFhLE9fzs3L71ZKjlOARJA+MlnCSeccjMQUH9jLXMJo9pZ3a6b9GQWVFJcWYvdamFQou8Eyp35egZiaEzDAAJgbOJY7p2o14af3vQ0q4+698AIiqzrTXHw22bHEhNq91QivNEBu3RuzN5IQVUBkYGRTEya2Oz5gwKuwVUdhyWghBe3/L1Fr3Gw+CDXL7meTTmbCA8I56WZLzF31LW8dP04zh2WyFu3TGJi35jmb5TlnoHo0foAwqzE2FmwE4er+d/yr52oO1O+t+Fw/f4Vwy6FyN5QkQ+b32nZiztq4P25OvDvPRnOeqD+cXuI3jUWYOU/ul1fiK25W3lt+2sAPDy5fZYu/KWUIjE0kTN7n8lto2/j2bOf5Y0L3iAmKIbdhbu5aelNTTYKEyc+CSD85LOEE06JNtbH8+5IabfYPf+ItecyxhZ3A6mhSeHYbQ3/ujpdTvYU7gH0EkZjrh18LZf0vwSX4eLe7+6tS9wbPFt/X/1iixoSmRUBi7dmkVta3fTJfjKXL2b2mUmAxffyhbc1+0upzLoKhYVF+xexPGN5k+evP7bes4FSclgyb85+01MmOqV/LK/+dDxj3E3CmmQYdUsYSaObP78RqRGphAaEUumobLpzqNv5wxOJDgngWEkV3+7x+jtmtdV92K9+oWUf9l8/qntIBEXBFf/2vfnXhFvBHg45O2Dv0obHu4j30sWF/S7krD5nNXn+p5uPcvNr6+tvjd5BBkYPZMGsBcQHx7OvaB83Lb2pQ/KiRPcgAYQfDMOoy4GIa2wG4tSowgAYlhRBoM1CUUUtB/LKm+4F0Up1G2j5DszSS9KpclYRYgshJSKl0fsopXhoykMMjx1OUXURv13+W51BPvEXEBCqPxD3Lmt2PCOSIxnr7ktQb6vpNnK4HHx16CsAzks9r/nznS5W7cvDVZnCBb2vAeDR1Y9SVOV7aWXR/kX8YtkvKKkpYVT8KN6a/Rb9o/q3brDFmVBZqLuuJvie9WkJq8XqqZppal8MU6DN6mnq9fa645Ipx/5ULx/m72v+w37P0rreEZf+E6J6+z4vOAom3KQff//39u142QYvb36Z/cX7iQ2K5Q8Tml662Hq4mLvfS+PrXTksXNPxJcgA/SL7sWDWAhJDEjlQfICblt5Ednl28xeKE44EEH4oKK+huFJnGPc9PgfiFFzCsNssjHbvjLnxUKGnlDOnsv1mIJrLfzAbSA2OGYxFNf3XOdAayNMzniY6MJqdBTt5dPWjGCExMH6uPuG7v7XoQ8LcB+LjtCPtlu+x/th6CqsLiQqMYmKP5pcvthwp9rSv/uPpd9M/sj/5Vfk8sfaJeue5DBfP//g8D6x8AIfLwfmp5zP/vPnEBftfteFhLl/EDwVbYNPnNqOlHSlN17qTKb/ZlUNOiVcJYWA4jNMtmfmhicZSxUfgo9v040m3w5DZTb/g5F/pvhCH10P6yhaNsSNty9vGf7b9B4CHpjxEVJDvyiSA0qpafv32Jmqd+u/o59s6rnroeCkRKSyYtYCk0CTSS9KZu3QuWWWd9/qic0gA4QezgVRyVDDB9uMyz71bWZ9CxqTof8B+zChs91JOl8totoTTu4FUSySFJfHUmU952vb+d9d/dca9NRAOr6vbObQJ5w1PxG6zsD+3nF3HSlv4bprmWb5ImYnN4mM6/Tjf79Eli9MHxhESEMQT05/Aqqx8kf6F517VzmrmfTePV7a8AsCtI2/lr2f8lSBbUNsG2w7LF6YRsS3rSGkamBjOuJRonC6D/208bofUSbfpWZFDK+HIpoYXOx3wwS16E7Wk0XDuo82/YHgijLlBP175jxaNsSM9tvoxXIaLC/pewDl9zmn0PMMwuP+jbRzKr6BnZBBWi2LXsdJGN1DrCL3De/ParNc8janmLp3L4dLO3dVWdCwJIPzQaP4DnJIzEADj+jSsxGivHIhDBRWUVjkItFkYmBjm8xxPBUYjCZS+TEyayN3j7gbgb+v/xvryTD0FDvBd83sghAcFcNZg/V4/3Xy0xa/bmFpXLV9nfA00X31hOr599fC44dw88mYAHl/zOHsL93Lz0pv5Iv0LbMrGY1Mf486xdzY7S9Ming6UrU+gNJmVGHsL91LtbNkavTkL8e76TFzeyZSRyTDiCv3YV3vrb/8CGT/ovIYrF7R89mTanXpDuP3f6LyJLnK49DA7C3ZiVdZmqy7eXZ/Jp5uPYrUonv/JWKb0iwXgi+3HOmOoHj3DevLarNfoE96HI2VHmLt0bsOmbuKEJQGEH8wKjP7HV2DAKRtAmJUYe7LLCA/Q/0i1VzdKc/ZhaFIEAT42iTIMw9MDwlcJZ1NuHHYjs/vOxmk4uefbe8gYfZX+7fXgt5C5vtnrzV0iP9uS1eZljHVZ6yiqLiImKIbxieObPb+kqtZTRurd/+G2UbcxOHowRdVFXPnplWzO3Uy4PZx/nfsvLht4WZvGWE87VGCYeoT2ICYoBofh8ASDzblwVBLhgTYyCipYfeC4LH+zpHP7x1Dk9UF14Fu9RAVw8TMQ60f+R3QqjLxSP/6+62YhVmfp6qHR8aOJCWq8UmZPdimPfKpzSu45bzDjUqKZNaIHAF9s69wAAvR/4wWzFpAakcqx8mPM/WKu7LNxkpAAwg/eTaTqcVTrRjZwSiVRAsSFBZIaq1sol5TpDbPaawZi6+Gmt/A+UnaE0ppSbBYb/SP9SwhUXVTvSQAAIABJREFUSvHI1EcYHD2YgqoCrvv+btYMc//234KdGM8ZmkBwgJWMggpPoNNa5pLDuSnntmj5YvX+fJ/tqwOsATwx/QlsFhsuw0WvsF68NfutFpWEtlhZrt4/BAU9RrT5dkqpevtitESI3cYlY3QA9/bxiaxJo3Q7asMJa152jzkHPrwVMPRMkxkM+GP6b/X3nZ9C7h7/r28HZvmx9wZrx6uscfKrhZuoqnVxxqB4/u+MfoBedlMK0jKLyCr2o1dGO0kISWDBrAUMiBpATmUONy29if1F+zt9HKJ9SQDhh7omUsdXYHj1PTjFciCgbhbiWL5uZtReAYS5hXdzDaQGRg1stGtjU4Jtwfxz5j8ZGTeSkpoSbqvYzsKICIw9X9T9lt2IELuNs4fqpNEWt7bO3qGT+LzUOv1fvljp3r7bV/vqwTGDeeqMp7huyHUsvHAh/SL7tWxsLXXMvXwR218nLrYDfzpSmq6doHtCfLk9m4LymvoHzS6Sm17X1SIf/Z9uSx0/FGb9pXWDTBjqLvk1dHfKTuZwOViTtQbQm1415pFF29mbU0Z8eCD/uHo0FovuzJoQHsR49/+nS7tgFgIgLjiO+efPZ1D0IPIq87hp6U2eEmxxYpIAooVqnS4y8nUL3YY9INy/gQZGtLqt74nM7Aex/5h+73mVebiMtjXecbkMth/VgVljJZxmBYa/yxfe4kPiWTBrARf3uxin4eLJ2CgeiYuhxpzubsLFo/QOlYtbsoyx81P45xR4ZgS8/RPY9zW4XKzJWkNJTQmxQbGMTRjbojE3t333OSnncP+k+5uc5m61dly+MPk7AwG6nHZEcgQ1ThcfbjouMW/ATIgfAjVl8NrFOnfBFgxXLdANolprus6bYcs7UFRXRmoYBlnFlfWbW7Wz7fnbKa0pJdwezvDY4T7P+STtCO9uyEQpePaa04gLq5/jcf5w9zJGJ+dBeIsJimH+efMZGjOUgqoCbl56s6cRnDjxdGgAoZSKUUotVEqVKKWKlFLzlVK+s+EaXquUUp8rpQyl1KUdOc6WyCyowOEyCA6w0iPiuCz2aq8A4hTk2Vgrw4VC4TScFFQVtOmeB/PLKat2EBRgYYCvnBP8r8BoTKA1kCemP8E94+/BguLD8DBuLlpL3uG1TV43Y3ACoXYrR4oqPVtq+1RyFBa5fys2XLB7Mbx1OTw/lqXrngH08oW1BcFnRn4F6fkV2CyKyf06IEBojqcCo/0CCPMDMb0k3a8upuYsxDvrM+sHcN7trbPd5aGz/9qmnhUA9J4AqaeDywGrX+BwYQUvLt/H+c98x5Q/f8Mv3thQP6mzHZnLF5OTJvv8e5KeV879H+r3esdZA5jqY28UM4BYd7CA/E5oKtWYqKAoXj3vVUbEjqCouoibv7y5RX1ARPfT0TMQC4HhwLnARcAZwCstvPYuoHt0bqEugbJvXKhnWtDjFE2gNA1MCCc80EZ5DUTa9YdaW0s5zQZSw3tGYvORQAmtq8BojFKKnw3/GS/OfIn/z955h0dZZv/7fqakd0IIIQkJJaH3DjYUAQUEFbC3XXctu7p+XXdX17rq6upPXcu6dl0VG6KIimDBBkiRXgIESGgJpJHepry/P555J20mmZnMJEGf+7rmmmHeMs+EZN4z53zO50RiYGtIMJesusmZ5XBFiNnoHHX92XY33Rh2u/QdqDkJPUfADWtku2FwFJaTOawqle9h+tHdcGxTm+v8cb/8uY5KjSUyxPuyTbvJ918Lp05sSCzJEdIgypsLyQUjkgg1G9lfUMmmQ83mLgxbAOGOSaJDLoaRV/plrZXj5OyMuvWvM+dfn/D4yr3sOyFLm9/sKeD57/b75XWaowcQrsoXdVYbf3h3M1X1Nsalx3HL2f1dniMlLoyhvaKxa/DV7s41dooOjualc19iePfhzmm57c1aKjqegAUQQoiBwAzgt5qmrdc0bTXwR+ASIURSG8eOAG4HrgvU+rzFqX9w2cKp21j/OjMQRoNgRKr0aQgW8r69nRhtGUgV1RRRWFOIQJARm9Gu12rMlF5TWDTxn6TVWzih1XP18itZkbPC7f56N8byHfmuv32u+4/s7DCHScvkxCEw819w+x5+OvNWKowGulutjNz5Gbw8FV46E7a8DfXVLc9Fg/+DL+O7201tOZQ4hG+J/gsgoMFQalex5wFEZIiZWY4yUgtnSlMwXPwqTLpFdl0I4eIMnlFrsfHZ9jx++7+fGbnIwnZ7OsHUca1pBRP6xPHohUN5YI7Mojz51T7WHvBxrLgbKusr2VYotSeuBJSPLN/DzmPlxIaZeeaSkW4DbqChG6MTyxg6kUGRvHDOC5iEiRPVJwIyyVcRWAKZgZgIlGqa9nOj574G7MB4dwcJIcKAd4CbNU1r87dcCBEshIjSb4B/lF3NcDtEC371GQhoKGPYLPLH394Pgx1tCCj1umladBph5nbUtV2QnnE+i4IzmVxdQ629njt+uINnNj/j8hvSaRnxRIaYOFFex8bcZmWb/O3wtcOsaMYjEN/om2FQOCtNNgCmpU3HOGwhGIOkz8AnN8OTA2HFXVDU8I3WarOzxnFxOi2jlVHbgeKEQ6MQ1QvCu/n11E5HykLPHCl1LhknPSE+35HndIl1kn46nPugV2LP8vpyjlYcxWqz82N2Ibd/sI0xD33NH97ZwtdZJ7DY4JPISwG4OXwV7101mEvGpXL1pDTmj07GrsEt725t6pLZTjYc34BNs9E7qje9Ino12fblruO84Rjs9sSC4SRGt24SpgcQa/YXtfx5dQIRQREkR8rsk2rtPPUIZACRCDS5imiaZgVKHNvc8RSwVtO0Tzx8nTuBska3gFidNXhAKBMpV+gBRHmlvJi3J4Cw2TV25smfqbsWTn+WL1wRdcbf+M+JQq4tl5mnl3e8zK2rbqWyvrLJfsEmo7O23KQbo75auh7aLTBgFoy6uslx9bZ6Vh1eBcD0IVfBhS/B/2XBOQ9ATCrUlsrsxXOj4cPrwGZl29EyKmqtRIea3QZWASUA5QsdPYDYWey5kBJkKad/QgS1FjvLPDT10jSNguoC1uWv452sd3ho3UP8ZuVvOOuDs5j87mRmfjST0S9fylVvrGLJ5qNU1lnpFRPKTWf25cvbTuee2++A+AwMdeXw82vO8/7jgiEMSIykqLKOP767BavNPyn5tXlrAZjYs2n24VhpDXd8KP9Prj8tnakDerR5rr7dI+ifEIHFpvHtHjd/o5oGBXvkGPP6wDtXpkWlAVIDozi1aLvpvBlCiEeBv7axm0+f6kKIOcBUYKQXhz0CNHZ3iSQAQYTbFk741dpYN2ZESgxCQEVVGMFh7QsgPt2WR3W9jZgws+uMDzQYSAUogKD3RIy9J/N/h9bQP3ky91dm8d3R77hi+RU8M/UZUqNSnbvOGtaTDzcd5Yud+dw3e5BMIX91DxTthYhEmP1MixT62ry1VFoqSQhNYETCCPlkeDxM+ZNsQ9z/DWx8BbK/hJ1LoFs/VtvmAzC5XzeMzXU4HYHuQOnHDgydgXEDMQgDBdUFnKg6QY/wti+GILUrl4xL5cHPdvPehsNcOaFhoJqmaRypOMKB0gMcLDvIwbKD5JTlkFOWQ6Wl0u05NU1gD91NRJ8jTIi6md+Nncno1Nim2qfJf4JPbpJTXMffAOYQQoOM/OfyUcx5djXrc0p46ut93DG9fQJfcK1/sNjs3PLuFspqLAxPifHqdWYMSSR71X6+2JnP3JGOjEZdpSy1ZX8lf/fKHP4aE26GGf9s93tojbToNDgKuWW5AX0dhf/xJQPxBDJAaO12EDgOJDQ+UAhhAuIc21wxFegLlAohrEIIq+P5JUKI71wdoGlanaZp5foN8M9wgkaUVVsoqpS95ukqA+GSyBAzmT0i0awyiPJVA2Gx2fn317I3/PrT+ri9UOoljAHd2v8B7ZbT/wzA7F1f88YZT5EQmsCBsgNc+vmlzg91gMn94okNM1NUWc/6nBLY+4W8+APM+6/LdL9uHnVu2rkt7aUNRsg4Fy7/QOomAH54nKJdcly3u/bNgBOADgydMHMY/WL6Ad5nIS4c2Ysgo4FdeeXO0hfA/T/dz/kfn88t397Cvzf/m2UHlrGjaAeVlkoMwkDvqN6cmXIm1w25jrvHP0BS9V+p2Hs/ISf+TGJoGpgqWFf9KD8UvY5NszZ90WELIDoFqgpg69vOp/t2j+DRi+TP5z/fHnD/Ld9DjlQc4XDFYUzCxNjEsc7nn/pqH5sOnSQy2MSzl4x0OereHbKMoXFs32bqf/g3/G82/CsN3rsMNr0ugwf9d3L3JwGfQqoyEKcuXgcQmqYVapq2p41bPfATECOEGN3o8KmO13TXH/coMAwY0egGcBtwrbdr9RcHHNmHxKgQIoJdJG1UAAHIMobd4gggfOzCWLLpKLnF1cRHBHHt5DSX+5TXl3O0UiaZApaBAOhzFiSNAmsNQ/et4t1Z7zpNp278+kZe2fEKNrsNs9HgrC1/t2mH1DCAbCXsO7XFaetsdXx7RAYDbZpHDb0Yhl8Gmp0bSx4lisom9tUdhqUWCh1W0wEoYUAjIaWXLX2x4UFMd/z8390ovzmfqDrBJ/tlFXRA3ABmps/k5hE388QZT/DxnI/ZePlGPpv3Gc9OfZYbh93CJz8msfdQLDEhkbxzzTw+vXAxCzPlmPQ3dr3B5csvb/oN2WhuMKxa84wc1OVg9vAkrpooMyG3fbCVY6W+Oz/qgeqw7sOICJLZuB+zC/nv91LM+uhFw0jt5qEGqK4C9nzOoJ/vZV3on/jMeAdBq+6DnB9kqS02DcZeD5d9AH/eL70zyo82aF8CRFp0GgCHyg8F9HUU/idgGghN07KAFcDLQohxQojJwHPAe5qm5QEIIXoJIfYIIcY5jjmuadrOxjfH6Q5rmtZpCpuD7iysdX7lXRg6o1JjnRkIX0oYdVYbz3yTDcCNZ/YjLMh1hW1vyV4AksKTiA4OYNAmBJx+h3y84RUShJnXZ7zOnL5zsGk2nt78NNetvI5jlceYNSwJgZ0zs+6D6mLoMRTOvtflaVcfW02VpYrE8ESGdffg2/x5j1EdnkqSKObpiP+REhvqxzfpIQW7pf9BaJwUUQYAb0d7N+ZSx4CtZVvzqK63siR7CTbNxugeo1k8ezGPnf4YNwy/gXPTzqVfbD+CjNI11Wqz88d3t7D2QDHhQUb+d+04MnpEEmIK4e4Jd/P0WU8THRxNVkkWCz5bwMfZHzd4Toy8EsLiofQQ7PqoyXr+fv5AhidHU1pt4eZFm6m3+qaHaO4+WVBRy23vb0XT4PLxqZzv6EJplcPrHVmGdHjvMsTmN0jUCqnTzGSFj4UZj8IfNsEtW+H8/wcZ02XWrM+Z8vi97ruQ/IGegcirzKPW6qH41G5z26mk6DgC7QNxObAH+AZYDqwGftdouxnIBPwro/czrU7hBJWBcDC6d0MAUVJbgsXuncr7vQ1HyCurJTEqhMvHp7rdz1m+aKeBlEdkzICEwVBfAetfItgYzEOTH+Ifk/5BmCmMzQWbuWjZRZywr+bGsK+ZzDZsxmBZenAz7dFZvujtonzhiuBI/pd0NxbNyFnWNbB1kT/foWc0Ll+0oyWyNfTR3ruKd3ntCTChTzd6dwujss7KJ9uOsGTfEgBnFsEVdrvGXz7czle7TxBkMvDK1WMZntJ0bPzU1Kksmb2EcYnjqLHWcO/ae7njhzuk4VVQGEy4Qe7445PS88NBsMnIc5eNIirExNYjpTzyhfdui83tq/cXVHL9/36mqLKeAYmR3DNrkGcn+vLvjbIM6TDud2Sf8zrD615iQeUd1I35HcT3a/n/mjlD3u9d7vXavSEuJI7IoEg0NA5XeDip850F8Hhf2PhqwEssCvcENIDQNK1E07TLNE2L1DQtWtO06zRNq2y0PVfTNKFp2netnENomrY0kOtsC7PRQGJUiOspnKBElA56dwsjNiQGTTOgoVFcU9z2QQ5q6m08961sWfzj2f0IMbt3ZXR2YLTDwtpjDAY4/Xb5eP1/oa4CIQTz+s/jwzkfMjJhJFWWKu776R7y4pdTajDwScJNkOA6uKm11vLdke8Az2dfALyXl8CTVimiZPlfmrR3dggBsLBuTr/YfgQbg6mor/B65LPBIFjoyEK8vvlzCmoKiAuJ45zUc1zur2kaD3y6i4+2HMNoEDx/2Sgm9nXdmtojvAcvTXuJW0fdikmYWJm7kouXXcyWgi0y5R8UCYVZsO+LJselxIXx5AJZhX19TS5f7PBwZooD3b46KiiKFZuNzHz6B7YdLSMi2MRzl41q9W/ESV0lHNssH1+/Cm7dCuc9Tt9J84iOiqKizsra/W7+TjMcAUTeZqgInG+EEIL0qHTAQyFldQns/xos1fD5/8G7l8ohb4oOR83C8IDbpmWw7q6zuWZSmusdnBmIGNfbfyUIIRjduxua1XsviDd/yqWwoo6UuFDmj05pdd+Ad2A0Z9Bc6NZPukk2attLiUzh9emvc8vwmzBp8E14COf3SuG+0hjqrDaXp1p9bDU11hp6hvd0zoBoi0PFVRwqruZVbTbW3lPAUgVLfgPW+rYP9hd6B0aA9A8AZoPZmVXyVkgJcPHoZIwGwRGbHE52Uf+L3A5Ze/Krffzvp0MIAU/MH845g1rv+jAajPx26G95c+abJEckk1+VzzUrruG/e9/FOsYhz1p6E+z7sslx5wzqwe/PkAPN/vLhdnKLPG+L1Ns3a8r78J9vc7DYNM7K7M7yW06jX4JHEwHgyDo5mTQmFXo1yNEMBtEwG8PdcK3IRKkBAtgX4DKGQwfhkZAyd7W8D42V3in7voD/ToLsrwO2PoVrVADhBcJd6laVMJw0LmN4KqSsqLU4RWG3np3RqqK81lrrNJzpkBIGyK4IfZDS2ufA0iCKMxqMXJ+fy9t5+aRb7ZSbNEh8lT99dR811pbiOb18MT1tuvvfp2b86Ji+OaJ3N0wXviQ/OPO3wrcPte99eYrdBiccwsYABhDg22AtnYTIECZl2jGF7wcEF2e4Htv98g8HeXaVzOD8Y87ghlZGT9bXfSiLZy9mVp9Z2DU7z297nt9YDpLXa6T07nhnPqx6WP7MHPz53EzGpsVSUWflpkWbqbW4Di4bc7yslre3fgVAxck+JEaF8MIVo3jtmrGeiyYBctfI+95TWmya4Qggvtx93L1nReZ58j7AOojeUVJ06lEGIucHeT90Plz/LSQMkt0wiy6CL/7a5O9TEVhUANFe7HapboZfvYgSmnZieJqBeG11LqXVFvp2D2deGx/m2SezsWk24kLiSAhLaHVfvzJsAUSnyg+qzW81WtBXsP4FBtdbeH/y4/QLlmnf1QWfsPCzhU2smWusNXx/9HvAu/KFc3x3v3iI7gVznpUb1jwNB75t5xvzgKJssNaAORzi+gb0pXwZ7d2YmESH8W31QOKCW2YV3t94mIeXywzWHdMzuXJimtevEREUwSOnPcI/p/yTcHM4m4u2c3F4HWuHz5M7/PAYLLoYqmRpwGw08Oylo+gWHsTu/HIe+NR9l4nVZue11Tmc/dQXlGkyqL544Fl8ffsZzBjS0+Og08khRwCR1jKAGJceR2yYmZPVFjY0d1HV0XUQB78LqGjRq1bO3B/lffrp0hr++lXSiwNg/Qvw0llwvOnvT0F5Lf/5dj+3vreF/DIVYPgLFUC0l7pynDO/fuUaCJDW08ImMzEHStp2BiytrueVHw8CslTUlkFS4/KF1x+m7cFohim3ysdr/i3LB5WFMm0NMP4GQgecx90T76L68HVo1ihyynK44vMreGn7S1jtVn48+iM11hp6RfRyO5K5OS7tqwfOhtGOtPnHNzgvVAFDF1AmDpWakACiZyD2lOzxWoRbY61hc4n81l5dNI6VzeY9fL49nzsdEyt/f3ofbjqzfcHQ7L6zWTxrMUPjh1JhqeR+Wx6WeS/I9scDq+DF0+GoHJCWGB3C05eMRAg5t6PFCHJg65FSLvjPGv7x2W7qTNkIYadnWAqPXHCm6/bxtqivatA/pE1usdlkNDiHwa10V8boMUT6XVhrpNFUgHCWMMpym05WbU7FCUc7sYDejvdkDpXzZS5fIgeoFWbBy2dhX/scq7Lyuf7Nn5n46CoeX7mXT7bm8ZcPt7f+GgqPUQFEe9HLF6YQMLfuQ/9rIMRsdGYG9hUfa3P/F384SEWdlYE9ozhvSNstac4AoiMElM0ZcYV0liw/BtvehWV/kBmJhEHSghoYlRpDonk4lQdvZVjMaVg1K89ueZZrV1zL+3vfB6R5lKfBj1v76un/hPgMqDwu1xHID0Sn/iFwAkqd1MhUIoMiqbPVsf+kd0LRFTkrKK8vJ8KYgK0qg/caDdj6bm8Bf3p/C3YNLh2Xwt9mDvBLAJoSlcJr018jLiSO/Kp8voqMhuu/kZma8qPw+gxnp8CU/vHc6piU+fePd7LvhMxcltVYuHvpDuY9v4ZdeeVEhZiYNFQGhWemtswceMyRDbLzIioZYnq73GWm429uxa7jrofBCdEgptz7RcvtfiI1MhWBoMJSQUmtm2wINGQfEodCWLNx9v3PgZt+oib9XLDVY/jy75jemc+23Xuw2TVG944lyGTgx+wiPtnqme25onVUANFeVAdGC/rFyQmVxypaHxlcWFHHG2tyAbh9WkbLMeku2FMsOzA6TP/QGHNIg3nQF3+VwjK9ZdMRPAohZG++LZzoit/w8JSHCTeHs7VwKxuObwBgRtoMj1/yx2ypI5nSL75pdiYoDC56VYrI9i6Hn1/1z3t0RQAtrJsjhHC2c3rrB/HB3g8AmJ8xHyEM/HSwmNyiKjbmlnDD25uw2DRmDevJQ3OH+jV7FWIK4dIBcsDWG7veQEsYBL/7TmaKbPWyU+DjG6C+mj9O7c9p/eOpsdi48e1NfPDzEc5+4jveXncYTZOumqv+fCZFNvneXY3v9hhn+WKy29bbSf26EREsh8FtPVrq+jx6GWPfiiatqv4kxBRCUoT83Gi1jKHrH9JPb/J0vdXOFzvyuer9gwzaczV3WX5DjRbE6cYdfBdxF2vnVrPkxknOAO4fn+3mZFUHipB/oagAor0oAWULhvaUXRQlda2LKJ//bj81FhvDU2I4e2DbegaL3cK+k9LmusM6MJoz5lpppqQLJKc9AD2aliP0EdOr9hZwdvJ5LJmzhFEJUs3eJ7qPV2vXBZQux3f3HObMfLDy71DgvddAm2haIw+IwAoodXwZ7b2raBc7i3diNpi5ZtgCTnfYfT/yRRbXvbGRWoudMzO78+SCEQGZI3JJ5iWEGEPIKsli/fH1Ug+14C2Y9iAII2x/D145B+PJgzy1cAQ9ooI5UFjFXz7cTlFlPX27h/PO9eN5cuEIarQCjlQcaWFf7TW57vUPOsEmI1MHyL89t90YaadBUARUnoD8Lb6vpw2cOojWhJSN9Q9Ij55Hlmcx6dFvuHHRZn7YV4imCQ6lL2DdtI+xJw4jzFpG0orfwrJbuH58DzJ7RFJSVe/Uwih8RwUQ7cUZQKgMhM7E3rKn20IplXVWl/vkldawaJ3s9f/zuRkefSPMKcuh3l5PhLlhBHCHExQOkx1aiH7nwLjft9hlaK9oUuPCqLXY+WZPAb0ievHa9Nd4/uznef6c5z3+9rt0yzE2Hz4JwBRXAQRI8Vi/c8BaCx/+RlpO+5PSQ/J33GCG7h2T9fHFkbJxeSguJI5LHWO+V+46QUWtlXFpcfz38tFezYzwhpiQGOb2mwvAGzvfkE8KAZNvgauXydp8wS546Uzij37Nc5eNwmwUBJsM3DE9ky9uPZ1JfeX/sW5fPTxhOOFmN+Z1bWGpgWMOQWnvlvqHxswc0tDO6VIbYApusGQPYBmjzVbO0iNQchBNGHkzL4mFL/7E1Ce+58UfDlJUWU/3yGBuOrMv399xJot+O4GzpkzB8NtvHH+vAjb/j6BXz+Rf5/VCCPhw01HW7i8K2Pv5NaACiPbitLFWGQidAd3lxV0Ya9iY67qM8dy3+6m32RmfHufxbAfdQCozLtMzB8dAMekWuOoTWLjIpahQCOHMQnzmGDFtNBg5Lfk0ekV41jL4ydZj/N8H0rL4qom9SY5107pnMMDc/0J4d3mB+vo+396TO3QDqYSBYAry77ndoAcQB0oPUG1pW/lfVlfGFznywqY7T549sAfxEdIJdHBSFK9cM4bQIA+Ml9rBVYOvwiAMrMlb47RbB2QG4Pc/QMoEWfJ87zLGZj/NqtumsPqvU7n5rH5NAhs9gGg+vtsrjm6U5ZPInhDXp9Vdz8jsTojZwOGSanbnl7veqQPaORtnIKw2O9knKli2LY9/rdjDta9v4B/PvgjAVls69644zPqcEgwCpg5I4KUrR7P2b1P5y4wB9O7WKOgyBcG0f8ggLiIRivczomSlc2rrXR/v8KitVuEaFUC0F1XCaEFUUBQG5MVmTe7BFtsPF1fzwUYpcLv93EyPv5HrFtadVr7QMRjknIBWRLOzhsl67nf7Cqmo9a6b4JOtx7jt/a1Owd/9s9vo2IhIgAuel4/XvwD7Vnr1eq0SwAmc7kgISyAhLAG7ZneKZlvj0wOfUmurpX9sf0Z0l86PZqOBRy8cyiVjU3jzunFEhbg2lPInKZEpTufLN3e/2XRjVE+45jM5Hhtgzb9J+ewyuouyJrtZ7VbW58tZg+3SPzj9H9zrH3TCgkyc4ejwcduN0f9cOaHzxA6ZCfAjVXVW1h4oIuuw/HtafWg3g+5bybSnfuCWd7fw3+8O8O3eQgbVSy3OzqDhnDuoB3dMz2T1X6fy2jVjOXdwImZjK5ez9NPlYDqAsqPcMT2TxKgQcoureXZVtl/fz68JFUC0FyWibIEQgiizVEhvOdbSkvjf3+zDatc4PaM749LjWmx3R6d2YHjJwJ6R9OkeTr3Vzle7WxeTNqZx8HDJ2BQenjvUI3EpGefC+Bvl46Ww8ma+AAAgAElEQVQ3yXY3f+DswBjR+n5+RhdStuUHoWmas3yxMGNhk2D0nEE9ePSiYXSLcD2TJBBcO0S21y4/uJzjVc0uxkYzzPgnzH9Dagpyf4S35jXpoNlZtJMKi7SvHtTNw1kXrmgsoPQAfZrsil1uAojwbpAyXj72oyvlgcJKznj8Oy57eT1v/iBdOi2GIuqtFsKCjIxMjeGy8ak8eMFgZkfJrpwrL72Cl64aw81n9SMpxovBctGOsmf5MSJDzDxwgQzMX/z+IHuOu8m8KFpFBRDtRWUgXJIYLvvL9xUfbdIetr+ggqVbZHvnn8/N8Ph8ds3uTAt3SgeGl8gyhsxCfLbdsxkIy7blOYOHhWNS+Oc8D4MHnXPul3371UWw9Ab/KOY7YAaGK4Z298yRcsPxDeSW5xJmCmNW31kdsbRWGRI/hNE9RmPVrCzKcjP0bPA8aX5kCpWjsvUgjYbyxYSeEzAafCy5WGplCQOkANIDpg7ogdko2HeikgOFla538nM7Z0lVPde9sZGiyjriI4I5p39/TCIYIey8fUM/dt4/nY9vmsw/5w3lykw7wVV5UouTMsG3F4ySf4+Uy8+f6YMTmT64B1a7xt+W7MDmqo1V0SoqgGgvtY7WJxVANCE1Wn6jqbWXNvlAeuqrbOwanDuoB8OSPZ8dcrTiKJWWSoIMQaRHp/t9vYFgtkMH8WN2IWXVrZcxPt2Wx5/ekz4FC8Yk88iFXgYPIEsqF70qPUkOrIKsZb4uXVJxQvpMIFp0mgQa3WirrQBCzz7M7jvbd8Ghn7l2sMxCLN63mIr6Ctc7dc+UvgXQ5P9Jn3/RrvLFsU1SVBueIGe4eEB0qNkp4nTbjZE5U97n/tjgvusjdVYbv3/rZw4VV5McG8qKP53Gy1ePo1+sQ4BtONH0919v30weK1uYfSFKz0A0eEA8MGcIEcFyYuqi9Yd8O++vGBVAtBclonRJj3DZGmYwl7PpkOwk2JVXxuc78hEC/s+L7AM0lC8yYjMwGwJfz/YH/XtEktkjEotNa+GK2JjPtufxJ0fmYf7oZB69cJj3wYNOwoAGr4o1T7fPYErXP8T3h2APhzf5Cd3S+mjlUU7WnnS5T2F1Id8ellbeCzIXdNja2uK05NPoE92HKkuVc6y4SwZeIO+zPgWgvL7c2XkyMakdAkoP/B9cMWNIG8O14jOkINNWLwNUH9E0OUZ9Y+5JIoNNvH7NWKfgVRdSHipvdjHPadq+6RPRDgFzRT7YZHdYYnQIf5mRCcBjK/ZyvMzPXUy/cFQA0V5UCcMlCaEygBCmhgDiyS+lh8PsYUkMSPROM6J3YAzo1vXLF43RuzE+3e7a+e7z7fnc+t5WbHaNi0cn86+L2hE86Iz7vcxC5G1uuJj4QgcaSDUnKijKeTFx5wexJHsJVs3KyISRZMR6F5AGEoMwcM3gawB4K+stLDY32aeMc2VKvmgfFO5lY/5GbJqNtKg0p6mST+jTKtto32zOtEE9MAjYcayMoydddL8IARmOLEQ7ujH+/XU2n2zNk2PUrxhF/x6Rzm0uWzk1rZGBlGclGZeEdweDCTS7I7MmuXx8b0amxlBZZ+W+Zb7NYPm1ogKI9qICCJd0D5OqbmEqZ9Phk2w+fJJv9hRgEPCnc/p7fb4u04HhJbOGywvB2gPFFFfWNdm2fEc+t7y3BZtd48JRvfwTPABEdIcRl8nHa572/Tyd0IHRmNb8IKx2K4v3LQYaWje7Euf3OZ/40HgKqgv4IteNZiAkGvqeJR/vXsZP+VL/0K7yhbVeWliDx/oHnfiIYMamSVHzyl1uRLh6GSN7ZZOJo57y8ZajPP2N7Hp4aO4QTnMYfunoQaM+cReQAVZVgQyKk9thrGUwQqSug2gI6I0GwSMXDsVkEKzcdcJ9BkbRAhVAtBfVheESfR6GwVTOwcIqHlgmv0VeNCqZPt29S4drmtZkiNapRHp8OIOTorDZtSYK9y925PPHdx3Bw8hePH7xcP86JE78AyAg+0s4sdu3czg7MDrGgbI5TkfKopYZiO+Pfk9BdQFxIXFM6z2to5fWJkHGIC4feDngsLd2V0oaOFveZy3zj/4hb7N0SQ2LlzoLL2koY7gR/qZOkIFPdXGDUNNDNuSU8NcPG4aZXToutcU+LjMQevYhZbw0tWoPupCyrOkwswGJUfz+DOmXcd+ynV63Xv9aUQFEe1EZCJfoAYTRLMVW246WYTYKbjnb++xDYU0hJbUlGIWR/rHeH9/ZOLsxtskP5RU7mwUP8/0cPAB06wuD5sjHa5/1/vjaMjiZKx93QgkDmmYgml+A9bkXc/vNJcjYMQZX3jI/Yz6hplCyT2Y7g4MWZJ4PwsCRot1O++oxiWN8f1Fn+WKSV/oHnemDZQDx86GTFFS40AMYzdITAuQMFg/JKarid2/9TL3NzozBifx1hutSpJ6BKKktobze8eXMzfwLn9B1EOUtS4p/nNqftG5hnCiv4/GVe1tsV7REBRDtQdMaiShVBqIx3UNlalIz1IFBpu4vGZtKSpz3Cmq9fJEenU6I6dSbeKrrINbnFPPWukP84Z0tWO0a8wIVPOhMclhu7/gAytqejNqE446yQXRKy6mHHcSAuAGYhImS2pImngqHyg+xNm8tAsH8jPmdsjZPiA6O5qL+FwHw+q7XXe8U3g16T+anUPl73S77amgkoPRtimdSTCjDU2LQNPjSXRnD2c7pmQ7ipKNds7TawvDkaJ5aOMJtqS7cHO787Mgty5WtyLl+EFDqROkBRMu/hxCzkX/Ok+3Db6075NRuKdyjAoj2YKmR43JBZSCaEWYOI8IsSxXCVE6wycAfpnrWUtacU7V8oZMSF8bwlBjsGtyzdCdWu8YFI5L4f4EMHgCSR0PvKWC3wvr/endsfscO0HJFsDHYmXFqrINYvFdqH6b0mtJ5M1E85MpBV2IURtbnr2d3sZtS0qALWOsIINpVvrBZ4LB0sfRWQNkYfTaG286hfudIMWLRXig+0Oq56qw2fv/2JnKKqugVE8rLV7dtKd6kjFGwC2pOSuOtpJHevpWW6AFEsxKGzqR+8Vw8OhlNg7s+2kG9NTDTR38pqACiPejlC2GQv+CKJuhCyrF9DTw4dwg9onzLHjg7ME4BAyl36J4QAHOGJ/FEoIMHHX3w189vQI2bcc2u6MQOjMYMjXcYShVLdXyttZalB5YCXVM82ZykiCTOTZMp/zd2veFyH2vGDNbrAUR0O7pJ8raCpQpCYyHBdxfLGY4yxk8HiimtdjHyOjRGlkigVVdKTdO486MdbMgpISLYxGvXjCUhsu3PgCZTOfXyRe9JsnzSXlopYej8/byBxIUHsfdEBS//2NKKX9GACiDaQ2MBpQ/1xl86eivnlVNiWTAmxefzODswTgELa3dcOCqZwUlRXD4+lScXDMfUmm+/P+k/TV5M6itgk5s0uis6uQNDR9dB6IZSXx76krK6MpLCk5jSy7c0fUejG0t9mfsleZUtL1w764upNBiIttkYeLwdtfdDjdo3XQx585S0+HAGJEZitWs8/HkW24+WthSBOts53btSPrdqPx9tPobRIPjP5aPITIx0u2+T19cDiPLcBv8HLztK3NLMjdIVseFB3DtLBmBPf5NNTlGVf177F4gKINqDElC2ip6BKKwp9PkcZXVl5FXJD93MOO9V5V2FuPAgPr/lNB6eN7TjggeQga1uLLXuBbDWtb4/yNJcoeNC1skZCD2A2F28G5vdxvt7pPPk/Mz5vls9dzADuw1kfM/x2DQbb+1+q8V2XWA5oaYW457PfH+hxgO02slsR/vx4k1HmfPcGiY+soq7l+7g+32FMq2f6dBBHForSwzN+GTrMZ74Svq+PDBnsHNYlyc4SxhluQ2aDn/oH6DBjbLiuCz5uOGCEUmc1j+eequduz5qKeJVSFQA0R5UANEqeidGYbXvAYRevkiOSCYqSAlVfWLIxbL/vfI4bP+g7f1P7AbNJlsBo9phaOQH+kT3IdQUSpWlii9yv2B70XZMBhPz+s3r1HV5i24stSR7CWV1TSdwOts3a2plEFBV7P0L2KxweJ187OEArdb43el9eGrhcGYMTiQsyMjx8lreXneYq1/bwKgHv+LmFaWUR/aVvyf7v2ly7KZDJdzxocxg/XZKOlc4Rmd7SnqUtLM+XJ6Lva5cfr4mDm33ewIcZlJmQJNBhBuEEDw8dyghZgM/HSzm4y1eipB/JagAoj2oAKJV9ACioLrA53P8EsoXnY4pCCbeJB+vfabtIVvHdf+HYZ1emjMajM6plI9vfByAaanT6BbarTOX5TWTkybTP7Y/NdYapwEWSPtqvTwzMTJNXpC9aI90cnybLFOFRMuBau3EbDQwb2QyL1w5ms33TOP1a8Zy6bhUukcGU1ln5fPt+Sw6Kf9f1i5/i9fX5HD0ZDWHiqu4/s1N1FvtTBvUgzvP8/7vNikiCbPBTJ3dQr7JKMsX/so2GQxytDq0WsYASO0Wxg1n9AWkY6yiJSqAaA8qgGgVvR2rPSWMU70Do8sw6moIjpaufm2NY+6kCZzu0Ed7l9SWALBwQNcXTzZHCOHMQizKWkS9TYoTdfvq9Oh0eg6YK3f2ZQiaXr5IneS/i62DELORswYk8MiFQ1l/59l8fNMkbj6rL3ujpQZlSPUGHv50O1P+9S0zn/6Rkqp6hvaK5ulLRvgkFDYajKRGSpOpXLPZf/oHnaiGsd5toQ8Yy8pX475doQKI9qAHEMqF0iX+yED8EjowugQhUTBGivlY+0zr+3ayA2VzhnRv+EbdL6YfoxJGdeJqfGdm2kwSwhIoqini84OfAw3li4k9J8JAh/HXwe8aPls8pfEArQBiMAhGpsZyx/QB/Pv267GFxBElqrkyKR+DgOp6Gz2jQ3jl6jGEBZl8fp20KD2AMPlP/6DjdKNsO4AY0FMKP/PKajlZ5aIj5VeOCiDaQ52axNkauoiyoLrAJxFStaXa6YmvShh+YPwNYAyCwz81+AU0x2aFAodfQVcJILo1BBALMhcgTtGOJ7PRzBUDrwBkS6ddsze1r+6eCd36y2mX+770/MR2GxySczT8IaD0GIMR4wDZjXFfxiE23T2NF64YxUc3TfK5ZVunN9KyOic0EhL8/LfvQSunTlSImVSH+Z3KQrREBRDtQZUwWkUvYVjslhbCMU/Yd3IfGhrdQ7sTHxrv7+X9+ojqCcMc6X93WYiifWCthaBIiE3vuLW1Qq+IXgyNH0pyRDKz+8zu7OW0i4szLibcHM7BsoO8k/UORyuPYjKYGJs4VupNdPtxb8oYx3dAXZnMhHZ02cnpSvkFsWFmZgzpSc/o0HafNq1afl7kRsT5X4fjLGG4NpNqzqCeMsO8WwUQLVABRHtQNtatEmQMIiY4BoCCGu/LGKp8EQAm3SLv93wORdkttzsNpIa2y0vAnwghWHTeIpbNW0bEKW7YFhkU6bTffnLTkwCM6D6CMLPD4l0frrX/a6h3MVLbFXr5InUCGH0vG/hE36kyq3Uyp6H11w+kF0kDp1xDANono1pO5GyNQUmOACJPBRDN6RqfEKcqKgPRJu1p5dQFlCqA8CPdMyDzPEBzPWSrixhINUcIgdngByfCLsDlAy/HJExYHDb4Teyre46A6FSwVMOBb9ycoRl+9H/wmuCIBo3CPvemUl5hqSHtmLQuP2GtpNriYSDlKXoJw8P5MCoD4R4VQLQHFUC0SWMdhLfoLZx6G5/CT+j21tvehYpmA5O6WAfGL5HE8ERmps90/rtJACFEQxZitwdlDLu93QO02k2jMoZfOLKBGEstMXaZfThccdg/59XR52FUngBr28LIgY4MxP6CSuqsNv+u5RRHBRDtobGVtcIlup21t62cFpuF7FKZYlcZCD+TOgFSxkux3oYXG56327tsBuKXxjVDrsEkTPQI69Hy91vXQexb0fYFrmAX1JaCObzzRK+ZjmDoyAaoKmr/+RzTN9NM8nM1tyy3/edsTFi8LLugSXO1VthWuI0rvzyfqISNWO0a2Scq/buWUxwVQLQHlYFoE18zEAfKDmC1W4kMiqRXRK9ALO3Xja6F2PgK1FXIx6W5Mig2BkF3FbQFkozYDN45/x1en/F6S0vu5HEQ0UP+X+R83/qJchvrHzqpxBOd7HCK1CDbi+4RdzgGaOmW1jnlOe0/Z2MMBo9bOVfkrJBffuKWYgjOU2WMZqgAoj04AwiVgXCHnoHIKsliT8kerHarR8c5HSjjBp6ybXtdmszzoFs/+Tu8+U35nF6+SBjUeRejXxEDuw0kJdLFkDmDAQbMko/b6sbQB2gF2P+hTTLPk/e+uGg2pq4Sjm0CIC1xNBCADAQ0lDHaMJPKPimzoJqwEZL0PjuO+SHD8gtCBRC+YrNIoRNASEznrqULkxQhI/3thduZ/+l8Jr4zkau+uIrHNj7G8oPLOVJ+xKVHhBJQBhiDoSEL8dPz8vdZlS+6DroOYs/n0pvDFXa7HGYF0LuTJ5PqOoj9q8BS6/t5Dq8DuxViepOWOAJwTOX0Nx4EEJqmse+kHAhmFGaMISf4oajlMLRfMx3c8/MLorZRKktpINwyIWkCt4y8hfXH17O7aDcVlgq2FGxhS8EW5z7RwdEM6TaEIfFDGBo/lMHxg1ULZ0cwbCF8+7Dsh9/5UZdzoPxVkzYFQmOhulgaf6W7sHMu3CO3m0IhaWTHr7ExPUdAZE+oyIfc1dD/HN/OkyvLF6Sf1jDWuywXTdP8m4n0oIRRXFvMybqTGISBW4bey1Pb76HE9BXr8tYxIWmC/9ZyCqMCCF/RjZHM4R3fe30KYTaYuX7Y9Vw/7Hrsmp1D5YfYWbTTecsqyaKsrow1eWtYk7emxfGqAyOAmENg/O/hm3/AmqehyiF0TVQBRKdjNMuywNZFsozhKoBw+j+MlwPTOhODATKmw6Y3YPdS3wOIHD2AOIOUyBQMwkC1tZrCmkJnS7hfiG57HoaefUiNTOWKoXN4/IdPMcVs4K4f72bpvI/UdGBUCcN3lIDSawzCQHp0OrP7zubO8Xey6PxFrL9sPe/Neo+7x9/NBX0voF9MPwTym0b30O7ObyGKADHmOgiKkGr+qgIQBugxuLNXpYCG2RhZn7meoJrr0D90dvlCZ9AF8n7LW/DD4+CtfX1NaUMWLO00goxBTgG133UQHpQwdP1D/9j+BJkM9OYS7PXdKKw9wSPrH/Hvek5R1FdnX1EBhF8wG80M7jaYwd0GsxBps1xlqWJPyR6SwpNaKtQV/iU0FkZfAz89J/8dnwFBYZ26JIWDPmfK4K4iD/I2Q/KYhm2a1mEDtDymz1lw+h0yeFj1EFSfhHMf8tzR9NBa0OxyHohj5HZaVBpHKo6QW57LuJ7j/LdWD9wo9QxE/9j+AAxJSmDf7gVEpL3IZwc/48yUM5meNt1/azoFURkIX1E21gEj3BzO6B6j6RnRs7OX8utgwo1gcHyXUAZSXQdziCwLAOz+pOm2on2y5GQKgV6jO35trhACpt4N0x3fztf9Bz652b0ItDk5DfoHHb2V0+9CSr2EUVng1mtDz0BkxGYA0pHSXtObJGTHyYPrHmzXpOFfAgENIIQQcUKIRUKIciFEqRDiVSFEq2b2QojvhBBas9sLgVynT6gMhOKXQnQyDL9UPnZVa1d0Hno3RtanTUsCevkieSyYgjt+Xa0x8SaY+wIII2x7Bz64Eiw1bR/nMJBqPL67sZDSr4R1A2MwoMkMTzOsdisHSg8AkBHjCCAcjpQVx89iYNxAyurKuHfNvT5NGv6lEOgMxCJgMDANmAWcDrzkwXEvAz0b3f4SqAX6jAogFL8kzn8SrvkcRlzR2StRNKbfNJllOJkDJ3Y2PN/Z9tVtMeJSWPi2vEjvXQ5vX9zwmemKqqKG95fWEMSmR8uJsH7PQAjRahnjcPlh6u31hJpC6RUp9RIDE2UAceykhbvGPkiwMZg1eWt4f+/7/l3bKUTAAgghxEBgBvBbTdPWa5q2GvgjcIkQIqmNw6s1TTve6Nb17L+UjbXil4QpSF6MusgEToWD4Ajoe7Z8nPWpvNe0zh2g5SkDzoMrP5Kj4Q+thjdmQaUbS3s9o5IwGMLjnU/rGYhjlceot7U9t8Ir9DKGi1bOfaUO/UNMfwxC/k1Eh5npFSNHlddWx3Pb6NsAeOLnJwJjdnUKEMhPi4lAqaZpPzd67mvADoxv49jLhRBFQoidQohHhBBuVV1CiGAhRJR+AyLbv3QPUBkIhULREeizMfThWsUH5AwHY1BTYWVXJG0KXPOZnD9xfDu8Nh1KXQzHcqF/AIgPjSfMFIZds3Ok4oh/1+bMQLgIIEqaCih1Go/2vnTApUzoOYFaWy13/ninc7rqr4lABhCJQBOFiaZpVqDEsc0d7wBXAGcBjwBXAm+3sv+dQFmj21Hfl+wFysZaoVB0BBnTpci1MAuKshvsq3uNAXNo567NE5JGwHUrIToFSg7Aq9OhYE/TfVzoH0COcXcKKTuwlVMf5NcigGg02tsgDDw4+UEigyLZWbyTV7a/4t/1nQJ4HUAIIR51IXJsfvPZPlDTtJc0TVupadoOTdMWAVcB84QQfd0c8ggQ3eiW7Otre4WzC0NlIBQKRQAJjYX0M+TjrGUN5Yuuqn9wRXw/GUTEZ0rR4usz4KiceUF5vuwqQUDvSS0O1csYfh+q1YobZfMODB09A5HlGKqVGJ7IPRPuAeDF7S+yo3CHf9fYxfElA/EEMLCN20HgONDEOkwIYQLiHNs8Zb3jvp+rjZqm1WmaVq7fgAovzu07qoShUCg6Cr0bY/eyruf/4CnRveDaLyBpFNSchP/NhgPfNugfeg6XwVIzApaBcONGWVlfybFK+VyLAMKRgcg+UUm9VZp7zUyfycy0mdg0G3etvosaqwcdJ78QvA4gNE0r1DRtTxu3euAnIEYI0bhJearjNde7PLlrRjju871da0DRrayDVQChUCgCzIBZgID8rfKCZzDLsd+nGuHd4OplMqNiqYJ3FsDap+W2ZuULnfQo2YlxqPyQf9fipoSxv3Q/AAlhCUQ3+3xPjg0lMsREvc3OgcJK5/N/n/B3EsISyC3P5cmfn/TvOrswAdNAaJqWBawAXhZCjBNCTAaeA97TNC0PQAjRSwixRwgxzvHvvkKIe4QQo4UQaUKIOcCbwA+apm0P1Fp9QmUgFApFRxHRvWl6v9eoU9cxNDgSLl8srbpt9XDckfZ3E0AEzExKDyCqCsFa53y6uQNlY4QQDOzZIKTUiQ6O5sHJDwLw3t73WH1stX/X2kUJdM/W5cAe4BtgObAa+F2j7WYgE9D/EuqBc4AvHcc9ASwBZgd4nd6jAgiFQtGR6LMx4NTSP7jCFAzz34CRV8p/G4Mg1fWEy9TIVABK60oprS313xrC4qTHBjTxgtADiOblC53GQsrGTEqaxOUDLwfg3jX3+netXZSABhCappVomnaZpmmRmqZFa5p2naZplY2252qaJjRN+87x7yOapp2haVo3TdNCNE3rr2naX7qcD4TdDnUOqYXqwlAoFB3BwFkNj7uy/4OnGIww51mY8xwseFNmJlwQZg4jMVw27vk1C9HETKqhjOEcohXTMgMBTVs5m/OnUX8iPTqdwppC/rXxX/5bqxuOl9Xyyo8HA/467lCuMb5QXymHvoDKQCgUio4hOhkm/gEyZp76GQgdIWDUlZA5s9XdnJ0YZf7uxNB1EDIDoWma2w4MHT0DkXW8vIWNdYgphH9O+ScAy3OWc7zKm34Bz7HY7Lz8w0HOfuI7Hvo8i2/3ds5MDhVA+IJevjCYG1JgCoVCEWimPwyXvdf15l8EmN5RvYEADtUqk/ZBJ6pPUGGpwCRMThvt5vTvEYHJICittpBfVtti+5D4IYzpMQa7ZmdJ9hL/rhdYf7CY85/5kYeXZ1FVb2NUagyJUZ1zHVIBhC/UNfKAEKJz16JQKBS/cJwzMfxuJtV0Hoauf0iLTiPIGOTykGCTkX4JciakqzIGwMLMhQAs2bfEbw6VBRW13Pb+Vha+tI59JyqJCw/isYuG8eENk5zCzo5GBRC+oASUCoVC0WE4p3IGqhPDoYForQOjMe6ElDpnp55Nt5BuFNYU8v2R79u1RKvNzhtrcjj7/33Px1uOIQRcPj6VVbefwYKxKRgMnfclVgUQvqBsrBUKhaLD0Fs5D1ccxma3+e/EzQKItvQPOq0JKQHMRjMX9r8QoF3TOjcdOsmc59Zw/6e7qaizMiw5mqU3TebheUOJCXOdIelIVADhC8rGWqFQKDqMnuE9CTYGY7VbyatsOX7bZ6IdAURZ0wxEmwFEIyGlOy7KuAiBYF3+Oq9NsIor6/jLh9u46L9r2Z1fTnSomYfmDuHjmyYzPCXGq3MFEhVA+IIqYSgUCkWHYRAGUqOkH4RfZ2LoGYjqIiy1FU6NhbsWTh1dc3CouJqKWtcah14RvTgtWU4X/WDvBx4tx2bXeHvdIaY+8T0f/CyFnQvGJLPq9jO4YkJvjJ1YrnCFCiB8wWljrUoYCoVC0RE4dRD+FFKGxoJJTjQ9eHwjVs1KpDnS6TvhjtjwIHpGy86HPcfdj1/SxZRL9y+l1tqyY6MxO4+VMe/5Ndy9dCdlNRYG9oxiyY0Teezi4XSL6JpdNyqA8AWVgVAoFIoOJSBCSiGcZYzs41sAKaAUHnTXDXJhad2cyUmTSQpPory+nJW5K93uV1Zt4dKX1rH9aBmRwSbunz2IT/8wmdG947x5Nx2OCiB8QQUQCoVC0aE4Wzn93okhWzn3ndwLtN2BodOWkBLAaDAyP3M+0HoZY+nWY1TUWenbPZxv/nwG10xOx2Ts+pfnrr/CrogSUSoUCkWH4jST8rsXhDSTyq44DLQtoNRpq5VTZ26/uZgMJrYXbSerOMvlPu9vPALAFRN6kxDpuSlUrbUWi80/PhO+oAIIX1AZCIVCoehQ9FbOwppCKusrW9/ZGxwljH11RaVM6+4AABfeSURBVIAXAYQjA7H3RAVWm93tfvGh8ZyTeg4AH+xrmYXYeayM3fnlBBkNzB3Ry6ulP7XpKRYsvYDdxbu9Os5fqADCF2qViFKhUCg6kqigKOJCpCbA27bI1k+cRJnBQIFdjvTuF9PPo8NSYsMIDzJSb7VzsKiq1X0XZC4A4PODn7cIfvTsw7mDexAb7qG3w8lDbFp5O+/seYf9lUcoLXSd2Qg0KoDwhTpVwlAoFIqOJiBCyqhk9gWZAdl6GREU4dFhBoNwtnO2poMAGNNjDH2j+1JjreHTg586n6+12Fi6VXpQLByb0voL1pbD5rfg9fOpfWY49x35HIALq2qZZDd7tGZ/owIIX1AlDIVCoehwAiKkjEoi2ywvwG35PzTHKaRsQwchhGgiptSneK7YeZyKWiu9YkKZ3De+5YF2G+z/Bpb8Fv5fBiz7AxxazfMx0Rwym0kwR3D7VT/CoDlerdtfqADCF5wiSlXCUCgUio4iIF4Q0b3YFyRLB/3dTOB0hyetnDpz+s4h1BTK/tL9bC7YDMAHP8vyxfwxyU1nWhRkwVf3wlOD4e0LYcdisNZAfAbbJ9/E/2KkG+W9pz1KVETrnhWBxNRpr3yqYqkFm6yVqQyEQqFQdBy6kNKvGYiQGLKDpVFTRnA3rw7VMxBZ+eVomib9I+w2+PZhqDgOphAwh4IphEhzCOeF9WZJ+R7eX/cYvXvMIzRnH1MMQVzWMxzyquDwetj2LuRvbXiR0FgYcjGMuJT6HkO497OF2LEzq88szkg5w28/Bl9QAYS36OULBARFdupSFAqF4teEnoE4VH4Iu2bHINqfRLejke3QQGQYQr06NqNHJEaDoLiqnoKKOnpEhUD2V/DjEy73nx9kZkmvnnx1chd/3fYlrwY5ujcWN9vRYIaM6TD8Eug/HUwyQ/Lilmc5UHaAuJA4/jr2r16tNRCoAMJbdAFlcBQYVAVIoVAoOopekb0wCRM11hoKqgvatJz2hGOVx6gREGTXSK33zlMhxGykT3w42QWV7M4rlwHE4bVyY8oE6HOmLD1YasFaw2BLDUOqtrOTGp6P7MPskxp9YoxEGS1grZWzOYYthCEXQXjTbEhWcRav7ngVgLsn3E1MSOcP1VIBhLcoAaVCoVB0CmaDmeTIZHLLc8kpy/FLAKFP4OxjsWCqyPf6+EFJUTKAyC/nrAEJsgwBMOoqGHl5i/0XZH/MzrX38l5EBB9W3MmGW6aBydjqa1jsFu5dey82zca5vc9lWu9pXq8zEKiv0N5SWyrvlYBSoVAoOhx/t3Jmn8wGIKO+HsqPeX18EyGlpRbypECS1Aku95+RPgMjYRiCSpg4uIjgNoIHgNd2vMaekj3EBMdw5/g7vV5joFABhLcoG2uFQqHoNJxCSj91YugZiP71FijzIYBo3MqZvxVs9RAWD3F9XO5fXWug7uQoAGpDV7d5/v0n9/PC9hcA+Nu4vxEf6qLds5NQAYS3qBKGQqFQdBqNhZT+wJmBsFigPM/r43UzqdziKupzHPqH1Aly0qcLPt5yjNqS8QBsKfqJ/Er3ZROr3co9a+7BardyZvKZnJd+ntfrCyQqgPAWZWOtUCgUnYY/WzlrrbUcdgzR6l9fD+VHvT5HfEQwCZHBaBpU718jn3RTvtA0jfc3HkGr707vsGHYNTuL9zVvwWjgrd1vsbN4J5HmSO6ZeI9HY8Y7EhVAeIuysVYoFIpOQ89A5FXmUWutbde5DpQdwK7ZiQ2OJt5mh5qTUF/t9XlkGUMj5Pgm+USK6wBiy5FSsgsqCTEbuH7EFQB8lP2Ry4maOWU5PLflOQDuGHsHCWEJXq8r0KgAwltUCUOhUCg6jbiQOCKDItHQ2FOyp13n0ssX/WMzEPoMDB/KGIN6RtFH5BNiOSnNo3oOd7nfB47BWecN6cl5fc8hPjSe4tpivjnyTZP97Jqd+9beR729nklJk5jbb67Xa+oIVADhLcrGWqFQKDoNIQSn9ToNgPf3vt+uc+kCyozYTOnBAD6VMQYlRTHaIM9F0iin8VNjquqsfLpNBicLxqZgNpi5sP+FACze27SM8e6ed9lSsIUwUxj3T7y/y5UudFQA4S0qA6FQKBSdylWDrgJgRc4KTlSd8Pk8DRmI/hCVJJ/0MQMxVuwFwO6mfPH5jnyq6m2kdQtjfLocSz4/Yz4GYWDD8Q0cLDsIwJGKIzy9+WkAbh9zOz0jenq9no5CBRDeokSUCoVC0akMjh/M6B6jsWpW3t3zrs/nachAZEC0IwPhQytn727hjDXKcx2Pdl2+WOwcnJXizCgkhidyevLpcvvexWiaxv1r76fGWsPYxLFcnHGx12vpSFQA4S1KRKlQKBSdjp6FWLxvMdUW74WPRTVFlNSWIBD0jenbqIThfQBhrCkmXch2zG0io8X2A4WVbMw9iUHAxaOTm2xbmLkQgE8OfMLbWW+z4fgGQowhPDDxAb/M+ggkXXt1XRFVwlAoFIpO54zkM0iJTKG8vpxlB5Z5fbxevkiNSiXUFNquAIIj0r56n70X24paXlb1sd1nZSbIeRmNmJQ0ieSIZCrqK3hs42MA3DLqFlKiUrxfRwejAghvUQGEQqFQdDpGg5ErBspWyLd2v4Vds3t1vFP/ENNfPqGXMHzQQHB4HQA/2zOkI2UjLDY7SzbJoGT+mJZBgUEYmJ853/nvEd1HcNmAy7xfQyegAghvsFmhvlI+VgGEQqFQdCpz+80lMiiSwxWH+f7I914d20T/AA0ZiDLvuzAaAohMOROjEd/uKaCoso74iCDOHujay2Fuv7mEmcIINgbzwOQHMBrano/RFVABhDfUNfrFUCJKhUKh6FTCzGHMz5Df3t/c/aZXx2aXNurAgIYAorYU6qs8P5GlVs7AADZrmRRV1lFQ0WBwpZcvLhyVjNno+pIbFxLHe7PeY/HsxfSJdj1DoyuiAghv0AMIU6jLPl+FQqFQdCyXDrgUkzDx84mf2VW8y6NjbHYbB0oPAI0yECFREBQpH3tTxsjbIgdohSdg6JYOQFZ+BQAF5bV8u7cQgAUuyheNSY9OJz063fPX7QKoAMIblP5BoVAouhSJ4YlMT58OSC2EJxyuOEydrY5QUyjJkY26IqJ9KGMckeULUsczKEleG/Qyxoebj2Kza4zuHUu/hAjPz3mKoAIIb3AGEKp8oVAoFF2FKwddCcDKnJUcrzre5v66/qFvdN+mrZJRPggpD8sODFImNBntrWkai3+WgcjCNrIPpyoqgPCGWuUBoVAoFF2Nwd0GM6bHGI+NpZwjvOOaeTY43Sg9bOW02xtlICYwyDHae3deGRtySsgpqiI8yMj5w7qum2R7UAGEN6gShkKhUHRJvDGW0jMQzhZOnWhHOcPTEkZxtpzgaQqFxGHODEROURVv/nQIgFnDkggPNnn4Lk4tVADhDcrGWqFQKLokZ6ScQWpkKhX1FSzdv7TVfZ0ZiFh3GQgPSxiO9k16jQZTEAmRIcRHBGPX5OwLkIOzfqmoAMIblI21QqFQdEkMwuDUQryd9TY2u83lflWWKo5WygyDs4VTx1s3SocDJanjnU8N7BnpfNwvIYJRqTGenesUJGABhBAiTgixSAhRLoQoFUK8KoRoU4YqhJgohFglhKhyHPuDECI0UOv0ClXCUCgUii7LnL5ziAqK4kjFEb4/6tpYan/pfgC6h3YnNiS26Ua9hOFpAKFnIBpN4NTLGCDFk111FLc/CGQGYhEwGJgGzAJOB15q7QAhxERgBfAlMA4YCzwHeOdRGihUF4ZCoVB0WcLMYSzIXAC4N5Zy6h+aZx+goYRRWwZ1la2/WGUhlEgvCVLGOp/WhZQmg2DeqF5erP7UIyABhBBiIDAD+K2maes1TVsN/BG4RAiR1MqhTwHPaJr2qKZpuzRN26tp2geaptUFYp1eozIQCoVC0aW5dMClmAwmNp3YxK6ilsZSbvUPAMGREOz4fG8rC6F3XyQMgtCGTMaZmQmMSInh5rP6ER8R7NN7OFUIVAZiIlCqadrPjZ77GplJGO/qACFEgmNbgRBirRDihBDieyHElNZeSAgRLISI0m9AZGv7twuniFIFEAqFQtEVSQhLYGbaTMB1FsI5RMtVBgI8b+V0li+aXtKiQ80svXkyt01zEaD8wghUAJEIFDR+QtM0K1Di2OYK3QD8fuBlZAZjM/CNEMLN/zQAdwJljW4+TELxECWiVCgUii6PLqb8MvfLJsZSmqa1HKLVHKcbZVsZCF1AOaH1/X7BeBVACCEeFUJobdwGtHMtL2qa9rqmaVs0TbsN2Atc18pxjwDRjW7JrezbPlQJQ6FQKLo8A7sNZGziWKyalXf2vON8vqC6gPL6cozC6H5olSdulJYayJMDtJpnIH5NeJuBeAIY2MbtIHAcaDK3VAhhAuIc21yR77jf3ez5LCDV3YI0TavTNK1cvwEV3rwhr1AiSoVCoTgl0I2lPtz7odNYSs8+pEWlEWR0MxDRGUC0kszO2wJ2C0T0gNg0fy35lMOrAELTtEJN0/a0casHfgJihBCjGx0+1fF6692cPhfIAzKbPZ8BHPJmnQFB05SVtUKhUJwinJ78/9u71xi56jKO499fSy8s7W4LQi8UKmmBGi5BIUCjgSohRmIjGAQETXhlNBDkjUQSExASQUQggUZDU8QXal9wMyRcDJiKmFoRI+VWbi2U0kKBwm4LtKX08cX/TDs7nZ05x+zM6c75fZLJ7Myc3Tz79N+dZ/7XM5jbP5etn27l/lfvB5oc4d1MniGM9SvT/RGnQQ8v02ynI3MgIuJF0nLMpZJOlfRl0nLM5RGxEUDS4ZLWSDo1+54AfgVcIel8SfMlXQ8sAJZ1Is5Cdn4EkW1M4gLCzGy/Nk7j+P4Xso2lXkgbS7Wd/wD5dqOsHaB15MLRCHXM6uQ+EJcAa4DHgYeAJ4Ef1L0+gdTb0Fd7IiJuI81puBV4BjgLODsiXutgnPnUhi80Hib0tb7WzMxKt3jeYgYmDbBh2wZWvLmi/QoMgP42m0nt3t10B8oq6tgJHxGxBbi4xeuvA/v0/UTEjcCNnYrr/1a/AqPCXVZmZmNF34Q+LjjmApY+u5S7nr+LtYNrgZw9EDuG0rB145y3916G7R+mD5IzT+xQ5GODz8LIyyswzMzGnIsWXMQB4w5g9bur2bV7F1MmTGHWQS2O1540Ze/f+WbDGG/WHaA1fsLoBzyGuIDIa88ESq/AMDMbKw7rO4xzjjpnz+P50+a3P5+i1TDGCBtIVZELiLzcA2FmNibVNpaCNsMXNa12o6wVEBWfQAkuIPLb/mG6n+QeCDOzsWTBwQs4fVbaMfKEQ09o/w0jLeXcthk+WAdo2AFaVdWxSZQ9Z88kyt49293MrFfddMZNPPnWk8OGM0Y00hDG+roDtNwb7QIiNw9hmJmNWdMnT2fxvMX5Lh5pCMPLN4fxEEZe3sbazKwaBkY4D2PPBMrqHqBVzwVEXt7G2sysGvqbzIHY+TFsyg7Qcg8E4AIiPw9hmJlVQ62A2Ll179/+jf+B3btg6iyYNre82PYjLiDyqjUir8IwM+ttE/vgwOnp69owRv3+D96NGHABkd8OD2GYmVVG4zDGngmUnv9Q4wIiL0+iNDOrjloBMfTW8AO0vAPlHi4g8vIkSjOz6qhfyvnumvQhckIfzMyxEVVFuIDIY9cO2PVJ+toFhJlZ76vfjbJ2gNacUyp/gFY9bySVR633ATyJ0sysCup3o1y/K33t/R+GcQGRR20C5cSpMG58ubGYmVnn1Q9hfLAufe39H4ZxAZFH7SAtD1+YmVXDQNYDsWVt2v8BwRwfoFXPcyDy8AoMM7NqmTor3e/Ohi9mHOcPkQ1cQOThFRhmZtUysQ8OPHjvYy/f3IcLiDy8C6WZWfXUVmIAHLmwvDj2Uy4g8vAulGZm1dNfX0C4B6KRC4g8fJCWmVn11AqIqbNh4IhyY9kPuYDIw5MozcyqZ/rn0/3chT5Aqwkv48zjxAvTDNwZ3sLUzKwyvvg9+PTj9B5g+1BElB3DqJLUDwwODg7S3+8eAzMzs7yGhoYYGBgAGIiIoVbXegjDzMzMCnMBYWZmZoW5gDAzM7PCXECYmZlZYS4gzMzMrDAXEGZmZlaYCwgzMzMrzAWEmZmZFeYCwszMzApzAWFmZmaF9exZGENDLXfgNDMzswZF3jt78SyMw4ENZcdhZmY2hs2JiLdaXdCLBYSA2cDWUfyxU0lFyZxR/rljkXOROA+J85A4D4nzkIz1PEwFNkabAqHnhjCyX7hl1VSU9p4Dv7Xd6WS9zrlInIfEeUich8R5SHogD7li9iRKMzMzK8wFhJmZmRXmAiKfHcDPs/uqcy4S5yFxHhLnIXEekkrkoecmUZqZmVnnuQfCzMzMCnMBYWZmZoW5gDAzM7PCXECYmZlZYS4gzMzMrDAXEDlIukzS65K2S1ol6dSyY+omSddKiobbmrLj6jRJZ0h6UNLG7Hc+t+F1SbpO0iZJn0h6TNLRZcXbKTnycHeT9vFIWfF2iqSrJT0laaukzZIekHRswzWTJS2R9L6kbZLulTSjrJg7IWceVjRpE78tK+ZOkPQjSaslDWW3lZK+Ufd6z7cFFxBtSLoQuIW0pvdLwDPAo5IOKzWw7nsemFV3+0q54XTFQaR/78tGeP0q4Argh8BpwEektjG5O+F1Tbs8ADzC8Pbx3S7E1W1nAkuA04GzgQnAXyQdVHfNrcBi4DvZ9bOB+7ocZ6flyQPAUoa3iau6GWQXbAB+CpwMnAL8FfizpOOy13u+LXgfiDYkrQKeiojLs8fjgDeB2yPixlKD6xJJ1wLnRsRJZcdSFkkBnBcRD2SPBWwEfh0RN2fPDQDvAJdGxPLSgu2gxjxkz90NTIuIc0f8xh4k6VBgM3BmRDyR/fu/C1wcEfdk1ywAXgQWRsQ/y4u2cxrzkD23AvhvRFxZZmzdJmkL8BPgHirQFtwD0YKkiaTq8rHacxGxO3u8sKy4SnJ01oW9VtIfJB1ZdkAlOwqYyfC2MQisonptA2BR1p39kqTfSDqk7IC6YCC735Ldn0z6NF7fJtYA6+ntNtGYh5pLJL0n6TlJN0jq63Zg3SJpvKSLSL11K6lIW+i50zhH2eeA8aRPlfXeARZ0P5zSrAIuBV4idUVeA/xd0vERMRaPqh0NM7P7Zm1jJtXyCKlrdh0wD/gF8LCkhRHxWamRdUjWE3kb8I+IeC57eiawMyI+bLi8Z9vECHkA+CPwBqmX7kTgl8CxwLe7HmQHSTqBVDBMBraReudekHQSFWgLLiCsrYh4uO7h6mxY5w3gAmBZOVHZ/qJhuOZZSauB14BFwOOlBNV5S4DjqcZcoFaa5iEi7qx7+KykTcDjkuZFxGvdDLDDXgJOIvXCnA/8XtKZ5YbUPR7CaO094DOgcebsDODt7oezf8iq6peB+WXHUqLav7/bRoOIWEv6v9OT7UPSHcA3ga9GxIa6l94GJkqa1vAtPdkmWuShmVXZfU+1iYjYGRGvRsTTEXE1abLxj6lIW3AB0UJE7ASeBs6qPZd12Z1F6raqJElTSF3Vm8qOpUTrSH8I6ttGP2k1RmXbBoCkOcAh9Fj7yJbt3gGcB3wtItY1XPI08CnD28SxwJH0UJvIkYdmahOwe6pNNDEOmERF2oKHMNq7hdQt9W/gX8CVpIkyvys1qi6SdDPwIGnYYjZpSetnwJ/KjKvTskKp/hPTUdnY5paIWC/pNuBnkl4hFRTXk8Z8H9j3p41drfKQ3a4B7iUVVPOAm4BXgUe7HGqnLQEuBr4FbJVUG8sejIhPImJQ0jLglmw2/hBwO7CyV2bdZ1rmQdK87PWHgPdJcyBuBZ6IiNVlBNwJkm4AHiZNjJxK+p0XAV+vTFuICN/a3IDLSW+eO0hdcaeVHVOXf//lpDfGHaS1z8uBeWXH1YXfexEQTW53Z68LuI70xrmdNOP6mLLj7mYegANJhcJmYCfwOnAnMKPsuDuQh2Y5CNKy3do1k0lvsFtI+4LcB8wsO/Zu5gE4AvgbqXjYDrxCKir7y459lPOwLGvvO7L2/xhwdpXagveBMDMzs8I8B8LMzMwKcwFhZmZmhbmAMDMzs8JcQJiZmVlhLiDMzMysMBcQZmZmVpgLCDMzMyvMBYSZmZkV5gLCzMzMCnMBYWZmZoW5gDAzM7PC/gcg9QN6KrrrugAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 600x400 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8UUDl6uTl6W5"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}