bicondtional_encoding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a47c95f5-c1ba-4a45-bf43-917fd3c30b38",
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "import re\n",
    "import os\n",
    "import nltk\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "SEED = 1013\n",
    "np.random.seed(SEED)\n",
    "#nltk.download('stopwords')\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.corpus import stopwords, twitter_samples \n",
    "from stance_utils import *\n",
    "#from parameters import *\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras import Sequential\n",
    "#from tensorflow.keras.layers import Dropout,Concatenate,Dense, Embedding, SpatialDropout1D, Flatten, GRU, Bidirectional, Conv1D,MaxPooling1D\n",
    "\n",
    "from tensorflow.keras.layers import RNN, Dropout,Concatenate,Dense, Embedding,LSTMCell, LSTM, SpatialDropout1D, Flatten, GRU, Bidirectional, Conv1D, Input,MaxPooling1D\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras import Model\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "stemmer = PorterStemmer()\n",
    "tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)\n",
    "stopwords_english = stopwords.words('english')\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import keras.backend as K\n",
    "from keras.layers import Lambda\n",
    "import random\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ef1eb2ac-3c04-4226-ad5f-ae132b6fae72",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = {'FAVOR': np.array([1, 0, 0]), 'AGAINST': np.array([0, 1, 0]), 'NONE': np.array([0, 0, 1])}\n",
    "classes_ = np.array(['FAVOR', 'AGAINST', 'NONE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fdc10ac3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "setting = 'cross' #'in' or 'cross'\n",
    "units = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "65230d80-7c1f-453d-a66c-c5043bf88588",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_file_m = '/data/parush/stance_mohammed/new_train.txt'\n",
    "test_data_file_m = '/data/parush/stance_mohammed/new_test.txt'\n",
    "TARGETS_m = [ 'Atheism','Climate Change is a Real Concern', 'Feminist Movement','Hillary Clinton', 'Legalization of Abortion', 'Donald Trump']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9f08aad6-d353-4f23-82d6-034c785aaff3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_test(train_file, test_file, target):\n",
    "    \n",
    "    sentence_maxlen = 0\n",
    "    target_maxlen = 0\n",
    "    x_s_token = []\n",
    "    x_t_token = []\n",
    "    y_train = []\n",
    "\n",
    "    print('Using', target)\n",
    "    with open(train_file, 'r') as trainfile:\n",
    "        for line in trainfile: \n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "            \n",
    "            \n",
    "            if line[0].strip() != 'ID' and target in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_s_token.append(tweet)\n",
    "                target_ = line[1].strip().lower().split()\n",
    "                if len(target_) > target_maxlen:\n",
    "                    target_maxlen = len(target_)\n",
    "                x_t_token.append(target_)\n",
    "                y_train.append(classes[line[3].strip()])\n",
    "    \n",
    "\n",
    "                               \n",
    "    \n",
    "    x_s_test_token = []\n",
    "    x_t_test_token = []\n",
    "    y_test = []\n",
    "    with open(test_file, 'r') as testfile:\n",
    "        for line in testfile:\n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "        \n",
    "\n",
    "            \n",
    "            if line[0] != 'ID' and target in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_s_test_token.append(tweet)\n",
    "                target_ = line[1].strip().lower().split()\n",
    "                if len(target_) > target_maxlen:\n",
    "                    target_maxlen = len(target_)\n",
    "                x_t_test_token.append(target_)\n",
    "                y_test.append(classes[line[3].strip()])\n",
    "\n",
    "\n",
    "    \n",
    "    return x_s_token, x_t_token, x_s_test_token, x_t_test_token, y_train, y_test, sentence_maxlen, target_maxlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "08ddd7e6-fbc1-4815-9486-7ec4fd80ddd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using Feminist Movement\n",
      "Using Legalization of Abortion\n"
     ]
    }
   ],
   "source": [
    "if setting == 'in':\n",
    "    x_s_token, x_t_token, x_s_test_token, x_t_test_token, y_train, y_test, sentence_maxlen, target_maxlen  = train_and_test(train_data_file_m, test_data_file_m, TARGETS_m[2])\n",
    "if setting == 'cross':\n",
    "    x_s_token, x_t_token, _, _, y_train, _, sentence_maxlen, target_maxlen  = train_and_test(train_data_file_m, test_data_file_m, TARGETS_m[2])\n",
    "    _, _, x_s_test_token, x_t_test_token, _, y_test, _, _  = train_and_test(train_data_file_m, test_data_file_m, TARGETS_m[4])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b5668309-53e2-466c-b413-0df0aac2eecf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words in vocab are 2349\n"
     ]
    }
   ],
   "source": [
    "vocabulary = build_vocab(x_s_token + x_t_token )\n",
    "vocab_size = len(vocabulary)\n",
    "print(\"Total words in vocab are\",vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "946d2dd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.asarray(y_train)\n",
    "_,balance = divmod(len(y_train),batch_size)\n",
    "balance = batch_size-balance\n",
    "y_train = list(y_train)\n",
    "for i in range(balance):\n",
    "    index = np.random.randint(1, len(y_train))\n",
    "    x_s_token.append(x_s_token[index])\n",
    "    y_train.append(y_train[index])\n",
    "    x_t_token.append(x_t_token[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c8215158-4421-4a4c-88c0-b60598dbf465",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_s = [tweet_to_tensor(each_s,vocabulary) for each_s in x_s_token]\n",
    "x_s = pad_sequences(x_s, maxlen = sentence_maxlen, padding = 'post')\n",
    "x_s_test = [tweet_to_tensor(each_s,vocabulary) for each_s in x_s_test_token]\n",
    "x_s_test = pad_sequences(x_s_test, maxlen = sentence_maxlen, padding = 'post')\n",
    "\n",
    "x_t = [tweet_to_tensor(each_s,vocabulary) for each_s in x_t_token]\n",
    "x_t = pad_sequences(x_t, maxlen = sentence_maxlen, padding = 'post')\n",
    "x_t_test = [tweet_to_tensor(each_s,vocabulary) for each_s in x_t_test_token]\n",
    "x_t_test = pad_sequences(x_t_test, maxlen = sentence_maxlen, padding = 'post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0decbff1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b255d44e-796b-498e-9736-ecf96a732ff1",
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffle_indices = np.random.permutation(np.arange(len(y_train)))\n",
    "x_s = x_s[shuffle_indices]\n",
    "x_t = x_t[shuffle_indices]\n",
    "y_train = np.asarray(y_train)\n",
    "y_train = y_train[shuffle_indices]\n",
    "y_test = np.asarray(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ef1fa903",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "embedding_matrix = get_embeddings('twitter',100,vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2609d376",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-30 22:08:58.917886: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /apps/cuda/cuda-11.2/lib64:/slurm/include\n",
      "2021-10-30 22:08:58.917940: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2021-10-30 22:08:58.917971: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (gaivi2.cse.usf.edu): /proc/driver/nvidia/version does not exist\n",
      "2021-10-30 22:08:58.918580: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "embedded_inputs = tf.nn.embedding_lookup(embedding_matrix, x_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2cf8efe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([672, 18, 100])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedded_inputs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "34a2b7b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "h_0 = tf.convert_to_tensor(np.zeros([batch_size, units]).astype(np.float32))\n",
    "c_0 = tf.convert_to_tensor(np.zeros([batch_size, units]).astype(np.float32))\n",
    "start_state = [h_0, c_0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7a14e559",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = embedded_inputs[:batch_size]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "67dabd51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([16, 18, 100])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5d6ae32a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "lstm = LSTM(units, return_sequences=True, return_state=True)\n",
    "fw_output, fw_h_0, fw_c_0 = lstm(inputs,initial_state = [h_0, c_0])\n",
    "bw_output, bw_h_0, bw_c_0 = lstm(inputs[::-1],initial_state = [h_0, c_0]) # feeding data backwords\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "47ee7754",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# opt = keras.optimizers.Adam(learning_rate=1e-3)\n",
    "# inputs2 = Input(shape=(sentence_maxlen), name = 'Input')\n",
    "\n",
    "# embedded_inputs = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1], weights=[embedding_matrix], name = 'Embedding')(inputs2)\n",
    "\n",
    "# lstm = LSTM(units,activation='tanh',dropout=0.1,name = 'lstm')(embedded_inputs, initial_state = [h_0, fw_c_0])\n",
    "# b_lstm = LSTM(units,activation='tanh',dropout=0.1, go_backwards = True,name = 'back_lstm')(embedded_inputs, initial_state = [h_0, bw_c_0])\n",
    "# cond_out = []\n",
    "# cond_out.append(lstm)\n",
    "# cond_out.append(b_lstm)\n",
    "# concat_output = Concatenate()(cond_out)\n",
    "# flat = Flatten(name = 'Flatten')(concat_output)\n",
    "# output = (Dense(3,activation='softmax',name = 'Dense'))(flat)\n",
    "# model2 = Model(inputs=inputs2, outputs=output)\n",
    "# model2.compile(loss = 'categorical_crossentropy', optimizer=opt, metrics = ['accuracy'])    \n",
    "# model2.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0184e3d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input (InputLayer)              [(None, 18)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Embedding (Embedding)           (None, 18, 100)      234900      Input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     (16, 32)             17024       Embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "back_lstm (LSTM)                (16, 32)             17024       Embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (16, 64)             0           lstm[0][0]                       \n",
      "                                                                 back_lstm[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (16, 64)             0           concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Flatten (Flatten)               (16, 64)             0           dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (16, 64)             0           Flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "Dense (Dense)                   (16, 3)              195         dropout_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 269,143\n",
      "Trainable params: 269,143\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "opt = keras.optimizers.Adam(learning_rate=1e-3)\n",
    "inputs2 = Input(shape=(sentence_maxlen), name = 'Input')\n",
    "\n",
    "embedded_inputs = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1], weights=[embedding_matrix], name = 'Embedding')(inputs2)\n",
    "\n",
    "lstm = LSTM(units,activation='tanh',dropout=0.1,name = 'lstm')(embedded_inputs, initial_state = [h_0, fw_c_0])\n",
    "b_lstm = LSTM(units,activation='tanh',dropout=0.1, go_backwards = True,name = 'back_lstm')(embedded_inputs, initial_state = [h_0, bw_c_0])\n",
    "cond_out = []\n",
    "cond_out.append(lstm)\n",
    "cond_out.append(b_lstm)\n",
    "concat_output = Concatenate()(cond_out)\n",
    "concat_output = Dropout(0.5)(concat_output)\n",
    "flat = Flatten(name = 'Flatten')(concat_output)\n",
    "output = Dropout(0.5)(flat)\n",
    "output = (Dense(3,activation='softmax',name = 'Dense'))(output)\n",
    "model2 = Model(inputs=inputs2, outputs=output)\n",
    "model2.compile(loss = 'categorical_crossentropy', optimizer=opt, metrics = ['accuracy'])    \n",
    "model2.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "eabf7a4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11904761904761904"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v_num = len(y_train)//10 \n",
    "_, b3 = divmod(v_num,16)\n",
    "v_split = (v_num  + (batch_size-b3)) / len(y_train)\n",
    "v_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca0f84be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "26fc23fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-30 22:09:00.757807: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "37/37 [==============================] - 3s 25ms/step - loss: 1.0678 - accuracy: 0.4189 - val_loss: 1.0555 - val_accuracy: 0.4250\n",
      "Epoch 2/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 1.0203 - accuracy: 0.4814 - val_loss: 1.0503 - val_accuracy: 0.4250\n",
      "Epoch 3/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.9761 - accuracy: 0.5051 - val_loss: 1.0271 - val_accuracy: 0.4375\n",
      "Epoch 4/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.9264 - accuracy: 0.5422 - val_loss: 1.0122 - val_accuracy: 0.4500\n",
      "Epoch 5/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.8668 - accuracy: 0.5845 - val_loss: 0.9917 - val_accuracy: 0.5125\n",
      "Epoch 6/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.7395 - accuracy: 0.6757 - val_loss: 1.0218 - val_accuracy: 0.5375\n",
      "Epoch 7/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.5921 - accuracy: 0.7635 - val_loss: 1.1774 - val_accuracy: 0.5250\n",
      "Epoch 8/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.4558 - accuracy: 0.8226 - val_loss: 1.2627 - val_accuracy: 0.5500\n",
      "Epoch 9/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.3258 - accuracy: 0.9020 - val_loss: 1.4921 - val_accuracy: 0.5625\n",
      "Epoch 10/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.2203 - accuracy: 0.9358 - val_loss: 1.5036 - val_accuracy: 0.5875\n",
      "Epoch 11/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.1606 - accuracy: 0.9662 - val_loss: 1.7716 - val_accuracy: 0.5625\n",
      "Epoch 12/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.1210 - accuracy: 0.9831 - val_loss: 2.0023 - val_accuracy: 0.5750\n",
      "Epoch 13/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.1203 - accuracy: 0.9730 - val_loss: 1.8695 - val_accuracy: 0.5375\n",
      "Epoch 14/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.1118 - accuracy: 0.9730 - val_loss: 1.9620 - val_accuracy: 0.5500\n",
      "Epoch 15/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0936 - accuracy: 0.9797 - val_loss: 2.0367 - val_accuracy: 0.5000\n",
      "Epoch 16/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0603 - accuracy: 0.9899 - val_loss: 2.1103 - val_accuracy: 0.5875\n",
      "Epoch 17/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0484 - accuracy: 0.9932 - val_loss: 2.1957 - val_accuracy: 0.5875\n",
      "Epoch 18/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0325 - accuracy: 0.9966 - val_loss: 2.5009 - val_accuracy: 0.5750\n",
      "Epoch 19/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0440 - accuracy: 0.9899 - val_loss: 2.3483 - val_accuracy: 0.6000\n",
      "Epoch 20/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0286 - accuracy: 0.9932 - val_loss: 2.7328 - val_accuracy: 0.5625\n",
      "Epoch 21/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0409 - accuracy: 0.9932 - val_loss: 2.6426 - val_accuracy: 0.5500\n",
      "Epoch 22/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0590 - accuracy: 0.9865 - val_loss: 2.4594 - val_accuracy: 0.5750\n",
      "Epoch 23/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0265 - accuracy: 0.9949 - val_loss: 2.8679 - val_accuracy: 0.5625\n",
      "Epoch 24/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0692 - accuracy: 0.9882 - val_loss: 2.3368 - val_accuracy: 0.5875\n",
      "Epoch 25/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0435 - accuracy: 0.9899 - val_loss: 2.5179 - val_accuracy: 0.5875\n",
      "Epoch 26/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0163 - accuracy: 1.0000 - val_loss: 2.6566 - val_accuracy: 0.5375\n",
      "Epoch 27/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0118 - accuracy: 0.9983 - val_loss: 2.7612 - val_accuracy: 0.5750\n",
      "Epoch 28/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0136 - accuracy: 0.9983 - val_loss: 2.8529 - val_accuracy: 0.5875\n",
      "Epoch 29/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0112 - accuracy: 0.9983 - val_loss: 2.7748 - val_accuracy: 0.5875\n",
      "Epoch 30/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0117 - accuracy: 0.9983 - val_loss: 3.0195 - val_accuracy: 0.5750\n",
      "Epoch 31/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0143 - accuracy: 0.9966 - val_loss: 2.8906 - val_accuracy: 0.5875\n",
      "Epoch 32/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0215 - accuracy: 0.9966 - val_loss: 3.0900 - val_accuracy: 0.5750\n",
      "Epoch 33/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0087 - accuracy: 1.0000 - val_loss: 3.0514 - val_accuracy: 0.5875\n",
      "Epoch 34/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0124 - accuracy: 0.9983 - val_loss: 3.1391 - val_accuracy: 0.5875\n",
      "Epoch 35/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0163 - accuracy: 0.9966 - val_loss: 3.0258 - val_accuracy: 0.6000\n",
      "Epoch 36/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0073 - accuracy: 1.0000 - val_loss: 2.9699 - val_accuracy: 0.6125\n",
      "Epoch 37/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0085 - accuracy: 0.9983 - val_loss: 3.2349 - val_accuracy: 0.5875\n",
      "Epoch 38/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0504 - accuracy: 0.9814 - val_loss: 2.9670 - val_accuracy: 0.5625\n",
      "Epoch 39/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0279 - accuracy: 0.9966 - val_loss: 2.8763 - val_accuracy: 0.6125\n",
      "Epoch 40/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0232 - accuracy: 0.9949 - val_loss: 2.6418 - val_accuracy: 0.6000\n",
      "Epoch 41/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0089 - accuracy: 0.9983 - val_loss: 2.6978 - val_accuracy: 0.6250\n",
      "Epoch 42/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0048 - accuracy: 1.0000 - val_loss: 2.7084 - val_accuracy: 0.6125\n",
      "Epoch 43/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0046 - accuracy: 1.0000 - val_loss: 2.7259 - val_accuracy: 0.6000\n",
      "Epoch 44/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0056 - accuracy: 1.0000 - val_loss: 2.8237 - val_accuracy: 0.6000\n",
      "Epoch 45/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0055 - accuracy: 1.0000 - val_loss: 2.8668 - val_accuracy: 0.6000\n",
      "Epoch 46/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0049 - accuracy: 1.0000 - val_loss: 2.9044 - val_accuracy: 0.6000\n",
      "Epoch 47/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0039 - accuracy: 1.0000 - val_loss: 2.9126 - val_accuracy: 0.6000\n",
      "Epoch 48/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0177 - accuracy: 0.9949 - val_loss: 2.9221 - val_accuracy: 0.5875\n",
      "Epoch 49/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0337 - accuracy: 0.9882 - val_loss: 2.7112 - val_accuracy: 0.6250\n",
      "Epoch 50/50\n",
      "37/37 [==============================] - 0s 6ms/step - loss: 0.0127 - accuracy: 0.9949 - val_loss: 2.9316 - val_accuracy: 0.6250\n"
     ]
    }
   ],
   "source": [
    "history = model2.fit(x_s, y_train, epochs = 50, batch_size = batch_size,validation_split = v_split,  verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "83413cc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA770lEQVR4nO3deXhU1fnA8e87k8keErJAgBDCpiyyr4pWxA03xLoXXFup2latS0tbW62t1V/3Wleq1g0V3Km7qIALoEFAQBEBWQIEkkBCQvbM+f1xbmAI2TOTyWTez/PMMzP33rn3vUO475zlniPGGJRSSoUvV7ADUEopFVyaCJRSKsxpIlBKqTCniUAppcKcJgKllApzmgiUUirMaSJQLSIib4nIFf7eNphEZIuInBKA/S4SkR85r2eIyLvN2bYVx8kUkRIRcbc2VhXeNBGEAeciUfvwikiZz/sZLdmXMeYMY8yT/t62IxKR2SKypJ7lqSJSKSLHNHdfxpi5xpjT/BTXYYnLGLPNGBNvjKnxx/7rHMuIyAB/71d1LJoIwoBzkYg3xsQD24BzfJbNrd1ORCKCF2WH9AxwnIj0rbP8EmCNMWZtEGJSyu80EYQxEZksIjki8ksRyQX+KyJdReR1EckTkX3O6wyfz/hWd1wpIh+LyF+dbb8TkTNauW1fEVkiIsUislBEHhCRZxqIuzkx/kFEPnH2966IpPqsv0xEtopIgYj8pqHvxxiTA3wAXFZn1eXAU03FUSfmK0XkY5/3p4rIehEpEpH7AfFZ119EPnDiyxeRuSKS5Kx7GsgE/ueU6H4hIlnOL/cIZ5ueIrJARPaKyEYRucZn33eKyHwRecr5btaJyNiGvoOGiEiis48857u8XURczroBIrLYObd8EZnnLBcR+YeI7BGR/SKyprZUJSJRzt/GNhHZLSIPi0iMsy7V+W4LnXP6qPZYyj/0y1TpQDLQB5iF/Zv4r/M+EygD7m/k8xOAb4BU4M/AYyIirdj2WeAzIAW4kyMvvr6aE+MPgKuAbkAkcCuAiAwBHnL239M5Xr0Xb8eTvrGIyNHASCfeln5XtftIBV4Gbsd+F5uASb6bAPc48Q0GemO/E4wxl3F4qe7P9RzieSDH+fwFwJ9EZIrP+mnONknAgubEXI9/A4lAP+BEbHK8yln3B+BdoCv2u/23s/w04HvAUc5nLwIKnHX3OstHAgOAXsDvnHW3OOeTBnQHfg3o2Dj+ZIzRRxg9gC3AKc7ryUAlEN3I9iOBfT7vFwE/cl5fCWz0WReL/Q+a3pJtsRfRaiDWZ/0zwDPNPKf6Yrzd5/31wNvO698Bz/usi3O+g1Ma2HcssB84znl/N/BaK7+rj53XlwPLfLYT7IXuRw3sdzqwsr5/Q+d9lvNdRmCTRg2Q4LP+HuAJ5/WdwEKfdUOAska+WwMMqLPM7XxnQ3yW/RhY5Lx+CpgDZNT53BRgAzARcNU5/wNAf59lxwLfOa/vAl6rG4c+/PfQEoHKM8aU174RkVgRecQp7u8HlgBJ0nCPlNzaF8aYUudlfAu37Qns9VkGsL2hgJsZY67P61KfmHr67tsYc4BDv0qP4MT0AnC5U3qZgb3Qtea7qlU3BuP7XkS6i8jzIrLD2e8z2JJDc9R+l8U+y7Zif2HXqvvdREvL2odSAY+z3/qO8Qvsxf0zp+rpagBjzAfY0scDwB4RmSMiXbC/9GOBFU71TyHwtrMc4C/ARuBdEdksIrNbEKtqBk0Eqm4R+xbgaGCCMaYLtigPPnXYAbALSBaRWJ9lvRvZvi0x7vLdt3PMlCY+8yS2GuNUIAH4XxvjqBuDcPj5/gn77zLM2e/MOvtsrFpkJ/a7TPBZlgnsaCKmlsgHqrBVYkccwxiTa4y5xhjTE1tSeFCcnkfGmPuMMWOwJZGjgNuc/ZUBQ40xSc4j0djODRhjio0xtxhj+mGrtW4WkZP9eD5hTxOBqisB+5+yUESSgTsCfUBjzFYgG7hTRCJF5FjgnADF+CJwtogcLyKR2GqHpv4ffAQUYqs7njfGVLYxjjeAoSLyfeeX+A3YKrJaCUAJUCQivbAXS1+7sXXzRzDGbAc+Be4RkWgRGQ78EFuqaK1IZ1/RIhLtLJsP3C0iCSLSB7i59hgicqEcajTfh01cXhEZJyITRMSDrQoqB7zGGC/wH+AfItLN2UcvETndeX220wAtQBG26svbhvNRdWgiUHX9E4jB/kpbhi2it4cZ2HrhAuCPwDygooFt/0krYzTGrAN+gm3s3YW9UOU08RmDrQ7q4zy3KQ5jTD5wIbaBtAAYCHzis8nvgdHYi94b2IZlX/cAtzvVKLfWc4hLse0GO4FXgDuMMQubE1sD1mETXu3jKuBn2Iv5ZuBj7Pf5uLP9OGC5iJRgG6NvNMZsBrpgL/j7sFVJBdhqH4BfYqt/ljnVYQuxpS2w389CbHJcCjxojPmwDeej6hCnMUapDsXpcrjeGBPwEolS4U5LBKpDcKoN+ouIS0SmAucCrwY5LKXCgt5JqjqKdGwVSAq2quY6Y8zK4IakVHjQqiGllApzWjWklFJhLuSqhlJTU01WVlaww1BKqZCyYsWKfGNMWn3rQi4RZGVlkZ2dHewwlFIqpIjI1obWadWQUkqFOU0ESikV5jQRKKVUmAu5NgKllGqNqqoqcnJyKC8vb3rjEBYdHU1GRgYej6fZn9FEoJQKCzk5OSQkJJCVlUXDcyeFNmMMBQUF5OTk0Ldv3RlWG6ZVQ0qpsFBeXk5KSkqnTQIAIkJKSkqLSz2aCJRSYaMzJ4FarTlHTQRKqcDLyYYtnzS9nQoKTQRKqcDyeuHFq2H+5VBTFexogqawsJAHH3ywxZ8788wzKSws9H9APjQRKKUCa+vHULgVSvNhwzvBjiZoGkoE1dXVjX7uzTffJCkpKUBRWZoIlFKBtfIZiEqEuG6wam6wowma2bNns2nTJkaOHMm4ceM44YQTmDZtGkOGDAFg+vTpjBkzhqFDhzJnzpyDn8vKyiI/P58tW7YwePBgrrnmGoYOHcppp51GWVmZX2LT7qNKqcApL4KvXoORMyAyDpY+ACV7IL5bUMP6/f/W8dXO/X7d55CeXbjjnKENrr/33ntZu3Ytq1atYtGiRZx11lmsXbv2YDfPxx9/nOTkZMrKyhg3bhznn38+KSkph+3j22+/5bnnnuM///kPF110ES+99BIzZ85sc+xaIlBKBc7al6C6HEbNtA9TA1/OC3ZUHcL48eMP6+t/3333MWLECCZOnMj27dv59ttvj/hM3759GTlyJABjxoxhy5YtfolFSwRKqfrt/gpWPwffuxWiE1u3j5XPQLeh0HMUiECvsbByLhz7U/s+SBr75d5e4uLiDr5etGgRCxcuZOnSpcTGxjJ58uR67wWIioo6+NrtdvutakhLBEqpI218Hx47DT69D+bNhOqKlu9j91ewY4UtCdRe9EfNgLyvYecX/o03BCQkJFBcXFzvuqKiIrp27UpsbCzr169n2bJl7RqbJgKl1OFWPAFzL4SkTDjtj/DdEnj1OtsNtCVWzQWXB4ZffGjZMedDRLQtFYSZlJQUJk2axDHHHMNtt9122LqpU6dSXV3N4MGDmT17NhMnTmzX2LRqSKlw8eV82LfFXpi79jlyvdcLH9wFH/8D+p8MFz4B0V3AWw0L74SEHnD63c07VnUlrH4eBp0JcT4NntGJMPgcWPsinP4n8ET74cRCx7PPPlvv8qioKN56661619W2A6SmprJ27dqDy2+99Va/xaUlAqXCwf6d8NpP4cO74V/D4clzYPU8qCy166vK4aWrbRIYcyX8YJ5NAgCTboIJ18LS++HTfzfveBvetvcNjLrsyHUjZ9jeROtf98eZKT/QEoFS4eCjv9keO1e+CVs/sdU2r8yCN7vA0PMgbz1sXw6n3gXH3XB4Q64InH4PFOfCu7dDfDoMv7Dx4618BhJ6Qv8pR67reyIk9rYxDLvAv+epWiVgiUBEooElQJRznBeNMXfU2SYKeAoYAxQAFxtjtgQqJqXCUuF2WPGk/XWeNck+Trj1UEJY8wIYL1z4JAydXv8+XC447xE4kG/bC+LToN/k+rfdvws2vgfH/xxc7vr3NeJSWPIXKMqBxAx/nalqpUBWDVUAU4wxI4CRwFQRqdsC8kNgnzFmAPAP4P8CGI9Sncuu1eCtaXq7j/5qf9WfcMuhZS4X9D0BznsYbt0At6xvOAnU8kTDJXMh9Sh4fiasewWMOXK71c/ZxDJyRsP7GvkDwNhtVdAFLBEYq8R563Eedf9qzgWedF6/CJws4TBOrFJtteFdeOR78PavGt9u3xZbTTP6CkjqXf82UQkQ07V5x41Jgpkv2sbmF66Ex06FbT5dHY2xx+szCVL6N7yf5L7Q53hY9Wz9yUS1q4A2FouIW0RWAXuA94wxy+ts0gvYDmCMqQaKgBSUUo37zBmL5rNHbKNvQ5b8BcQNJ9zsv2N36Qk/XgLT7rdVO4+fDs/PgPyNNins3WTvHWjKqBmwdzNsW+q/2FSrBDQRGGNqjDEjgQxgvIgc05r9iMgsEckWkey8vDy/xqhUyNm7GTYutPX8fY6H/90Iu748cruCTbDqORj3Q3vx9ieXG0ZfBj9bASfdDpsXwQPjbftBZAIMObfpfQw5FyLjw/KeguaIj49vt2O1S/dRY0wh8CEwtc6qHUBvABGJABKxjcZ1Pz/HGDPWGDM2LS0twNEq1cFlPw7ignE/ggv/a6t15s2E0r2Hb7f4z+COtN0/AyUyDk68DW5YabudFm6DERfb5c357NDz7D0FG98PXIyqSQFLBCKSJiJJzusY4FRgfZ3NFgBXOK8vAD4wRisMlWpQVZmtgx98NnTpYUfxvOgpe5/Ay9ccajzO2wBr5sP4H0FC98DHFd8Nzv473PKNvVGsuab8FlIGwrMX2RvQOrHZs2fzwAMPHHx/55138sc//pGTTz6Z0aNHM2zYMF577bWGd9CcjgGtFMj7CHoAT4qIG5tw5htjXheRu4BsY8wC4DHgaRHZCOwFLglgPEqFvnWvQNk+Wxqo1XscnPlneP3nsOhemPIbWPx/EBET2NJAfeJbWGJP6A5XvWFLNK/8GIp32ZgD3WfkrdmQu8a/+0wfBmfc2+Dqiy+6kJtu+jk/ueZK8FYzf97zvPPue9xwww106dKF/Px8Jk6cyLRp046cd9h4Yc/XEJtifwD4WcASgTHmS2BUPct/5/O6HGjizhSl1EGfPwqpR0PWCYcvH3MV5KyAJX8GT4wd/vn4n0NcanDibInoRJjxkm1fWHinvQ9h6j3134PQUZkaO9x2SZ4dkqPuo6aKUeku9uzazs61n5BXsI+u8dGkd43j57N/zZIlS3C5XOzYsYPdu3eTnp5++P7L9oG3qnlVbq2gdxYrFSp2fGFH8zzjL0f+YhaBs/4Ku9fC+7+3DbbH/Sw4cbZGRCR8/z+QkG6HsijJhfPmBG4sokZ+uTdbTZW9QJfuhWpnOOj9OfZZ3OCOAFcERETZRnG3hwvPP58XP1xJ7p58Lj53KnP/+x/y8vJYsWIFHo+HrKysI4efNsZO5hMRbbv6BoAmAqVCRfZj4ImzjbH18cTAxU/Df8+E8bMgNrl942srl8sOatelJ7zza3sX88yX7Hl1FMZrx0kq3QsVzgxnnljokgFR8fbC74posGrr4suu4pprriE/P5/Fb77I/GefpltKVzweDx9++CFbt2498kMV+21pI6lPwKrMNBEoVevN22DfVpgxv/2PXVMNNRUNF/1L98KaF+3QDI1NEpOUCTd+aS+qoerYn9i68Fd+bIfA8G0PCZbqCjuI3oECWw3k8tgG8pjkFiWqoUOHUlxcTK9evejRbygzvn8251x1E8OGDWPs2LEMGjToyA+V7LHHi0ny3/nUoYlAKbDF769es0X9mipwe9rv2Afy7fj/ezfb6pGjTjtym1XP2l+FzbkohnISqDX8Ylj+MCx/BMZcHZxzMgYqS2y9f0WRXRadCLGptoqmlb/O16w51Eid2ucolr72OKQNOiKhlJSUQOUBG0OXXrbLcIB0gr8YpfygYCOU7IaaSsj7pv2Ou2+LnQlsz1cQ3x2evRDe/8PhXQW9XlstlHkspLfqnszQIwITroP8DbD5g/Y9tjFwIM+OyFqwEaoO2H+bbkMhuZ8dnttfVTRxafYCX5xb//qS3ba9ITawAy5oIlAKYMtHh17n1nOXbiDs+tImgdICuHwB/HixHZrho7/C0+fZkgLA5g9taaEjVJG0p6HTIa4bLHu4/Y5pvDY5F+XYC3RSpk0AXXraBm1/c0XYZFBeaOeE8FVVbtsj4lID3oNKE4FSAFs+sb/6PLH1D9fgb999BE+cZS8EV78DmRNs1cC5D9gxfLYvh4dPgG3L4fPH7MVi8DmBj6sjiYiyw2NsfA/yv/XLLhu9X9V4Ye8We1Hu0hPSjra/xANdLRWXBrjsr39fB/YA4qxvvtbck6uJQCljYMvHtm9+96GBLxGsexWe+b692PzwXehWp4Fw9GXww/fsL9AnzoQNb8Hoy+2FMdyMvdoOk7H8kTbvKjo6moKCgvovlN4aW+qqKLLzI8S3w93YtdweO51n2T7bKA22nap0r+351YL2KmMMBQUFREe3rNutNhYrVbDJ9lvPOh5yE+1ELV5vYH4Jfv4ovHEr9B4Plz7fcBfPHsNh1mJ49Xr4brG9YSwcxXezE96vehZO/m3jPaaakJGRQU5ODkcMXGm8thquutz+exTlA/lti7ulvNWwfw/kltqxo8oLoXw/JLhg14EW7So6OpqMjJZN9qOJQKmtH9vnrONtI2D2Y1C4xTYM+tPnj8Ebt8BRU+GC/0JkbOPbxyTZiWCqypretjObcK2dwGblM7ZraSt5PB769u17+MKyQttja8cK+P4cGHZ622JtiwVz7JDi1y+F/1xsp/S8+Ol2ObRWDanQVrgN5l9ux2FprS0f26qAlAGQPtwu83c7wcpn4I2bYeDpcNHTzb+wi4R3EgDoOdL2mFr+iP8GXivda9tpnpoGO1fChU8Ef/7k439uh5F4arptJJ50Y7sdWksEKrStnmf7/29eBJc8Z+fjbYna9oE+k+xFt9sQ210v98ump25sri9fgNd+aidyv+ipwPQ+6ewmXAsvXAEb3oZBZzXvM5WldhC74l2w9zv7Y2HPOvtc2zAbEQ2XPFv/vRvtLbkfHHOBHTW2zyTIGNtuh9ZEoELb5kWQ3N92r3t6ui3eDz2v+Z/fu9leKLKOt+890fbmHn+VCNa9au+QzToeLp4buLFzOrtBZ9thHJY9VH8iKN1rxyjaudIOWle80/6q9hURY3sC9T8Zug22Sb/nyI41MN/3brW9pE78RbseVhOBCl2VByDnM5jwYzj+ZnjuUnjhKntzzsTrmrePLbXtAz6jefYYDpv8cBPT+jfhpR/aX3aXPq9VPG3hjrBzKyy8E3LXHrqxrqrcTtv50V+hohh6jLBzJWcdb4drTnAeSZnQNavjj2iadjT8cku7H1YTgQpd25baO4H7Tba9PS5/FV76Ebw9G/bvgFPuarrnz5aP7U1LqQMPLUsfbhsni3e3flKXbxfaqoz04TDjBTsgmWqb0VfAov+zQ0+ccx+se9mOtFq4DQacCqf+3nb/VS2miUCFrs2LbR/zzGPte0+MrYN/65fw6b/trF3TH2q4/70xsPUT267gO2RAD6fBOHdN6xLB7q/sRCtpR8NlL7epy6PyEZtsR15d/bwdbnvnSjsZzOWv2R8DqtW015AKXZsXQcb4w0fsdLnhzL/AKXfayVkW/7nhz+/7zpYcatsHaqUPs8+5q1seU2UpvHiVHZRs5su2T7jynwnX2lJgyR6Y/jDMWqJJwA+0RKBC04EC27PnpNuPXCdiu+LtXmcbEMdeDYm9jtxuyyf2uU+dRBCdaOuTW9Ng/PZsO1jZZa/Ym6GUf3UbDD/5zN7925HmKQhxWiJQoWnLEvvc78SGt5nyW3vX6Ad/bGAfH9shhdOOPnJd+vCWDzWx7hX44kmbhPpPadlnVfOlDtQk4GeaCFRo2rzITsfYc3TD23Ttc+iu1Lq/7g+OL3R8/UMKpw+3XUvL9zcvnn1bYMGNkDEOTvpNc89CqQ5BE4EKTZsXQ98TbLfCxpxwix2q4d3b7cW/VuFWO79s3faBWrUNxrvXNh1LTRW8+EP7+vxH23dSG6X8QBOBCj37ttiG3r6NVAvVikmCE2fbgds2Ljy0/OD9Aw0kgpYMNfHh3bAjG6b9y7YtKBViApYIRKS3iHwoIl+JyDoROWLgDBGZLCJFIrLKefwuUPGoTmTzYvvc3N4iY6+2t++/e7udGxic9oEUexdxfRLS7TjwTbUTbPoAPv4HjLmyZXc0K9WBBLJEUA3cYowZAkwEfiIiQ+rZ7iNjzEjncVcA41GdxXeLIT69/kbe+kREwim/t715Vj1jl2355ND4QvURsaWCxkoEB/Lh5R9D2mA4/Z6WnYNSHUjAEoExZpcx5gvndTHwNVBPHz6lfHzzth1EriFery0R9DuxZfPGDj4Hek+ED+62N3wVbTt8WIn69BgOeV8fmiykrg/vhrK9cMFjOnyECmnt0kYgIlnAKGB5PauPFZHVIvKWiNR7f7iIzBKRbBHJPmJSCdV5eL3w+k224XX3uvq32fMVlOa3/CYiETj9bjv934tX22UNtQ/USh9uJwypb4jrPV/Diidg7A91WAMV8gKeCEQkHngJuMkYU7cv3hdAH2PMCODfwKv17cMYM8cYM9YYMzYtrWXzd6oQsn25HQnU1MCr19neOHVtXmSfm9NQXFfGWBj6ffsrPya54faBWj1G2Of62gne/a3tvnriL1seh1IdTEATgYh4sElgrjHm5brrjTH7jTElzus3AY+IdKAxYVW7+upVcEfZydt3rYZP/nXkNpsXQcrA+u8Ubo5T7rDjE2VNanpAuq597cW+bjvBxvedoYJvs3PNKhXiAtlrSIDHgK+NMX9vYJt0ZztEZLwTT0GgYlIdmNdr2wYGnAyjZthf7ovutfX5taorYeunbRtbpmuWHaTstLub3tblssMd567xibPG9j7qmgXjZ7U+DqU6kECWCCYBlwFTfLqHniki14rItc42FwBrRWQ1cB9wiTG+d/2osJHzma0Wqu2CeeZf7Jg/r153qMvnjmyoOtD4sBLN0ec4e9dxc6QPtzeVeb32/cqnbTvFKb9veFRTpUJMwAadM8Z8DDTarcMYcz9wf6BiUCFk3au2WuioqfZ9XCqc/Xc7H/Gn/7J3CG9eDOJqupHXn3oMh88escNNJHS3vY56T4Qh57ZfDEoFmI4+qoLPt1oousuh5UPOtSWERffCUWfY9oGeo9p3aOfaO4xzV8Pqr2yvo0ufa1nXVaU6OB1iQgVfzud2jtkh049cd+Zf7dj+r8yyVUOt6S3UFmmDwOWBb96yQ1ofc0G7TiquVHvQRKCCb90rtifP0VOPXBeXCmf9zTbYeqvbfxKSiEg7Bv6aF+ygdafc0b7HV6odaCJQwVVbLdT/5IandBx6nn1EdYHeE9o3Pjg0Eumx19tJ0JXqZLSNQAVXbbVQU7+0z5tj6+c90e0Tl6+jz7J3Eh9/c/sfW6l2oIlABddXrzrVQmc0vl1EpJ2eMBgGnWkfSnVSWjWkgqc51UJKqYDTRKCCZ0c27N8BQ6cHOxKlwpomAhU8615tXrWQUiqgNBGo4PB6bftA/ylaLaRUkGkiUMFRWy1U301kSql2pYlABce6V+0du1otpFTQaSJQ7a9gk71Td8DJEJMU7GiUCnuaCFT7+u4j+M8UO1zE5NnBjkYphSYC1Z5WPAlPT4f47nDN+3YkUaVU0OmdxSrwvDXw3u/s6J39p8CFT2hPIaU6EE0EKrAqiuGla2DDWzDuGph6L7j1z06pjkT/R6rAKdhkZxjb87WdV2D8NcGOSClVD00Eyv+MgdXPwRu3gtsDM+bDgFOCHZVSqgGaCJR/le+HN2623UP7TILvzwneqKFKqWbRRKD8JycbXrwainLgpNvhhJvB5Q52VEqpJmgiUG3n9cIn/4QP74aEnnDVW5AZhJnElFKtErD7CESkt4h8KCJficg6Ebmxnm1ERO4TkY0i8qWIjA5UPCpAinfbewPe/z0MngbXfqRJQKkQE8gSQTVwizHmCxFJAFaIyHvGmK98tjkDGOg8JgAPOc+qPdRU2cbc1tq4EF65FipK4Jz7YPTlIOK/+JRS7SJgJQJjzC5jzBfO62Lga6BXnc3OBZ4y1jIgSUR6BCom5aOsEP45DN75Tcs/W1NlbxB75nyIS4NZi2DMFZoElApR7TLEhIhkAaOA5XVW9QK2+7zP4chkgYjMEpFsEcnOy8sLWJxh5YunoHiXvdt37UvN/9y+LfD4VPjkXzD2arjmA+g2KGBhKqUCL+CNxSISD7wE3GSM2d+afRhj5gBzAMaOHWv8GF54qqmGz+ZA5rFgvLDgBkgfDqkDG//cN2/Dy7Ps6wufgKHnBTxUpVTgBbREICIebBKYa4x5uZ5NdgC9fd5nOMtUIK3/HxRth+N+Bhf8FyKiYN5lUHmg4c988TQ8fykk94Vrl2gSUKoTCWSvIQEeA742xvy9gc0WAJc7vYcmAkXGmF2Bikk5lj0EXfvCUVMhsRec/yjkrYc3brF3BfsyBj76Oyz4KfQ7Ca56E7pmBSVspVRgBLJqaBJwGbBGRFY5y34NZAIYYx4G3gTOBDYCpcBVAYxHAeSsgO3L4Yw/H7rZq/8UOzfAontsddGYK+xyrxfe/Q0sexCGXQjnPggRkcGLXSkVEAFLBMaYj4FGu5EYYwzwk0DFoOqx7AGI6gIjf3D48u/dBtuWwZu3Qc+RkDYYXvsJrJkPE6+H0+4Gl05foVRnpHcWh5OiHXau4InXQVTC4etcbltF9PAJdsTQ5P6w6X04+Q44/ufaNVSpTkx/4oWTz+YABsbPqn99XKrtDVSUA5s/tDeJnXCzJgGlOjktEYSLygOw4gkYfA507dPwdpkT4NJ5tidR3xPaLTylVPBoIggXq5+D8kKY2IwmmYE6d4BS4USrhsKB12u7jPYcDb3HBzsapVQHo4kgHGxcCAUb4difaH2/UuoImgjCwbIH7DwBQ84NdiRKqQ5IE0Fnt2c9bF5kJ45vy5DTSqlOSxNBZ7fyaXB57FwBSilVD00EnVlNFXw5D46eau8RUEqpemgi6My+fRcO5MHImcGORCnVgTUrEYjIjSLSxRkl9DER+UJETgt0cKqNVs6FuG4wQO8LUEo1rLklgqudSWVOA7piRxW9N2BRqbYryYNv34ERF4Nb7xtUSjWsuYmgtvP5mcDTxph1NDGyqAqyL+eBt1qrhZRSTWpuIlghIu9iE8E7IpIAeAMXlmoTY2DVXOg1RucTVko1qbl1Bj8ERgKbjTGlIpKMTiLTce1cCXu+grMamhhOKaUOaW6J4FjgG2NMoYjMBG4HigIXlmqTVXMhIhqOOT/YkSilQkBzE8FDQKmIjABuATYBTwUsKtV6VeWw5gUYdDbEJAU7GqVUCGhuIqh2ppU8F7jfGPMAkNDEZ1QwfPMGlBfBqBnBjkQpFSKa20ZQLCK/wnYbPUFEXIAOXNMRrZwLXTKg74nBjkQpFSKaWyK4GKjA3k+QC2QAfwlYVKp1inbApg9g5KV2DmKllGqGZiUC5+I/F0gUkbOBcmNMo20EIvK4iOwRkbUNrJ8sIkUissp5/K7F0avDrX4OMDDyB8GORCkVQpo7xMRFwGfAhcBFwHIRuaCJjz0BTG1im4+MMSOdx13NiUU1oPbegT6TILlfsKNRSoWQ5rYR/AYYZ4zZAyAiacBC4MWGPmCMWSIiWW2OUNWvothWBe3Psc97voa9m+GEW4MdmVIqxDQ3Ebhqk4CjAP+MXHqsiKwGdgK3OkNXHEFEZgGzADIzM/1w2BD22X/g/T9ARd3bOAS6DdFZyJRSLdbcRPC2iLwDPOe8vxh4s43H/gLoY4wpEZEzgVeBgfVtaIyZA8wBGDt2rGnjcUNXRTG8fxekDoTB0yAxA7r0gsRekNBDZyBTSrVKsxKBMeY2ETkfmOQsmmOMeaUtB3ZGM619/aaIPCgiqcaY/Lbst1Nb+QxU7Icz/2LHEVJKKT9o9vjExpiXgJf8dWARSQd2G2OMiIzHVjUV+Gv/nY63BpY9BJnHahJQSvlVo4lARIqB+qpiBDDGmC6NfPY5YDKQKiI5wB04N6EZYx4GLgCuE5FqoAy4xLl7WdVn/RtQuBVO+2OwI1FKdTKNJgJjTKuHkTDGXNrE+vuB+1u7/7Cz9AFI6gODzgp2JEqpTkbnLA4FO1bA9mUw4Vq9Y1gp5XeaCELB0gchqguM0tnGlFL+p4mgoyvKgXWvwOjLIbrBJhmllGo1TQQd3WdzAAPjZwU7EqVUJ6WJoCOrKIEVT8Dgc6Brn2BHo5TqpDQRdGSrnrWTzBz702BHopTqxDQRdFTeGlj+EPQaC73HBzsapVQnpomgo9rwth1N9Njrgx2JUqqT00TQEZXtg0X32iknB+tookqpwGr2WEOqnRRsgmcvgn1b4YLHwK3/REqpwNKrTEfy3UcwbyaICy5/DbImNf0ZpZRqI60a6ihWPAlPT4f47nDN+5oElFLtRksEweatgfd+B0vvh/5T4MInIDox2FEppcKIJoJgqqmG+ZfBN2/aO4dPv0fbBJRS7U6vOsG09H6bBE6/R7uJKqWCRtsIgiV/Iyy6BwadDROvC3Y0SqkwpokgGLxeWPBTiIiCs/4GIsGOSCkVxrRqKBiyH4NtS+HcByEhPdjRKKXCnJYI2lvhNlh4p+0hNPIHwY5GKaXCJxEYYyitrA52EPC/G+3z2f/UKiGlVIcQNolg+dffcdmfnuA/ry9hX8Ee23Wzva16FjZ9AKfcqfMLKKU6jLBpI8jYu4yXuAWysQ/AuKORqHg7BWS3IZA5EXpPhB4jICLSvwEU58I7v4LMY2Hcj/y7b6WUaoOAJQIReRw4G9hjjDmmnvUC/As4EygFrjTGfBGoeDKGnQhdn2B3Xj5Lv97K5h25xNeUMbSLi+FdDQm718D61+3G7ijoNRp6T4D0YZDSH5L7t23O4DdvhapymPZvcIVNQUwpFQICWSJ4ArgfeKqB9WcAA53HBOAh5zkwEntB4nl0B6ZPhh2FZTz60Wb+8dl2ynbVMPnoG7h2chwTPBuR7Z/B9uWw9AHwVh3aR3x3mxBS+kPaIOg50pYeohKOPF51JWz7FDa8c2hugVPuhNSBATtFpZRqDTHGBG7nIlnA6w2UCB4BFhljnnPefwNMNsbsamyfY8eONdnZ2X6Lcd+BSp5aupWnl20lv6SCAd3iufK4LL4/uhexUm0v4Hs3QcFGO0R0gfP6wJ7aM7EX9x4joecoiIqHjQth4wdQWWxLF32/B4PPhlGXgcvtt9iVUqq5RGSFMWZsveuCmAheB+41xnzsvH8f+KUx5oirvIjMAmYBZGZmjtm6davfY62oruGNL3fx+CffsXbHfhJjPFwyvjdXHdeX9MToIz9Qkge7VsHOlc5jFRTvtOvi0+Go0+GoqdDvRIiM83u8SinVEiGfCHz5u0RQlzGG7K37+O8n3/H22lwiXC4uHJvBdZP7k9E1tvEPF++Gsr222ki7hiqlOpDGEkEwew3tAHr7vM9wlgWViDAuK5lxWcls31vKQ4s3MT97O/M+3875ozO4/qT+9Elp4Bd+Qnf7UEqpEBLM7isLgMvFmggUNdU+0N56J8fyp/OGsfi2k5gxIZNXVu1gyt8Wc/P8VXyXfyDY4SmllF8ErGpIRJ4DJgOpwG7gDsADYIx52Ok+ej8wFdt99KqmqoUg8FVDjdmzv5w5SzYzd/k2qmq8XH18X342ZQAJ0Z6gxKOUUs0VtDaCQAhmIqiVV1zBX9/5hvkrtpMSF8XsMwbx/VG9cLm0XUAp1TE1lgj0zqZWSEuI4v8uGM6r108io2sMt76wmvMf/pTV2wuDHZpSSrWYJoI2GNE7iZevO46/XjiC7XvLmP7gJ/zmlTXUeEOrlKWUCm9hM9ZQoLhcwgVjMjhtaHf+/u4Gnvh0C4kxHn4xdVCwQ1NKqWbRROAnXaI93DltKBXVNTy4aBPH9ErkzGE9gh2WUko1SauG/OzOaUMZlZnErS+sZsPu4mCHo5RSTdJE4GdREW4enjmGuKgIZj2VTVFpVdMfUkqpINJEEADdu0Tz0IzR5Owr48Z5K7XxWCnVoWkiCJCxWcncMW0oi77J458LNwQ7HKWUapAmggCaOSGTi8Zm8O8PNvL22txgh6OUUvXSRBBAIsJd5x7DiN5J3DJ/FZvzSoIdklJKHUETQYBFe9w8PHM0EW4XN89fTXWNN9ghKaXUYTQRtIMeiTH8YfoxrNpeyMOLNwU7HKWUOowmgnYybURPzhnRk38u/Ja1O4qCHY5SSh2kiaAd/eHcoSTHRfLzeasor6oJdjhKKQVoImhXSbGR/PmC4Xy7p4S/vftNsMNRSilAE0G7m3x0N2ZOzOTRj79j2eaCYIejlFKaCILh12cOpk9yLLe+sJrich2CQikVXJoIgiA2MoK/XTSSnYVl/OH1r4IdjlIqzGkiCJIxfbpy3eT+zM/O4Z11etexUip4NBEE0Y0nH8WwXon84sUvydlXGuxwlFJhShNBEEVGuLj/B6Oo8Rp+9txKqvSuY6VUEAQ0EYjIVBH5RkQ2isjsetZfKSJ5IrLKefwokPF0RH1S4rj3/GGs3FbIX9/RLqVKqfYXsKkqRcQNPACcCuQAn4vIAmNM3dbRecaYnwYqjlBw9vCeLN1UwCNLNjOxXwonDeoW7JCUUmEkkCWC8cBGY8xmY0wl8DxwbgCPF9J+e/YQBqUncPP8VewqKgt2OEqpMBLIRNAL2O7zPsdZVtf5IvKliLwoIr0DGE+HFu1x88CM0VRUe7nhuZU6SqlSqt0Eu7H4f0CWMWY48B7wZH0bicgsEckWkey8vLx2DbA99U+L50/nDePzLfv4h85qppRqJ4FMBDsA31/4Gc6yg4wxBcaYCufto8CY+nZkjJljjBlrjBmblpYWkGA7iumjenHR2AweXLSJJRs6b9JTSnUcgUwEnwMDRaSviEQClwALfDcQkR4+b6cBXwcwnpDx+2nHMLBbPDfNW8WOQm0vUEoFVsASgTGmGvgp8A72Aj/fGLNORO4SkWnOZjeIyDoRWQ3cAFwZqHhCSUykm4dmjqGy2sv1z6ygolqHrFZKBY4YY4IdQ4uMHTvWZGdnBzuMdvH22lyufWYFP5iQyZ/OGxbscJRSIUxEVhhjxta3LtiNxaoRU49J59oT+/Ps8m3Mz97e9AeUUqoVNBF0cLeedhTH9U/ht6+u1SkulVIBoYmgg4twu7jv0lEkx0Vy3dwVFJZWBjskpVQno4kgBKTGR/HgjNHkFpVz07xVeL2h1a6jlOrYNBGEiFGZXbnjnKEs+iaPf+rNZkopPwrYoHPK/2ZMyOTLnELu+2AjKfFRXHFcVrBDUkp1ApoIQoiIcPd5wygsreKOBeuI9ri4eFxmsMNSSoU4rRoKMR63i3//YBQnHpXG7JfX8OrKHU1/SCmlGqGJIARFRbh55LIxTOybwi0vrOatNbuCHZJSKoRpIghR0R43j14xlpG9k7jh+ZV8sH53sENSSoUoTQQhLC4qgv9eNY5B6V249pkv+GRjfrBDUkqFIE0EIa5LtIenrh5Pv9Q4rnric/7+3gbKKnWQOqVU82ki6AS6xkUy90cTOH1oOve9/y2n/H0xb63ZRagNKKiUCg5NBJ1ESnwU/750FM/PmkhCdATXzf2CGY8uZ8Pu4mCHppTq4DQRdDIT+6Xw+s+O565zh7Ju537O+NdH3LlgHTt1ghulVAN0PoJObO+BSv767jc899k2BPjeUWlcMq43Jw/ujsetvwGUCieNzUegiSAMbN9bygvZ25mfnUPu/nJS4yM5f3QGF43rTf+0+GCHp5RqB5oIFADVNV6WfJvH859t5/31e6hxRjF1CbhEELHDWLgEYiMjSImLJDkuktT4KJLjIkmJj6RHYjRHp3fhqO7xxEbqCCVKhYrGEoH+Tw4jEW4XUwZ1Z8qg7uwpLueNL3ex70AlBvAagzHgNWCM4UBlNQUllRQcqGR97n4KDlRSWFp1cF8ikJUSx9HdExjUI4FjeiYyaUAqMZHu4J2gUqpVNBGEqW4J0Vw1qW+LPlNV42VnYRnrc4tZv6uY9bn7WZ9bzDtf5WIMxHjcnDQojTOO6cFJg7oRH6V/XkqFAv2fqprN43bRJyWOPilxnD40/eDy0spqVm4r5O21uby1Npc31+QSGeHixKPSmDo0nUE9EuiVFENijAcRCeIZQI3XsG1vKclxkSTGeJrcflNeCU8v3cqaHUWM6p3Esf1TGNc3mS7RTX9WqVChbQTKr2q8hhVb9/Hmml28vTaX3P3lB9fFRrrpmRRDz6QYeiVF0ycljgFp8fTvFk/vrjFE1NOTyes15JdUsLOonPKqGrJS4ujeJarJhGKMIa+kgg25JQdLLt/kFrNhdzEV1V4iXMJxA1KZOjSdU4d0Jy0h6rBz+GD9Hp5auoWPvs3H4xaG9Ezk6137qaz24hIY1iuRif1SmNg/hQl9k7W9pBWKSqt48Yscvt61n2N6dmF0n64M7tFFe7QFSNAai0VkKvAvwA08aoy5t876KOApYAxQAFxsjNnS2D41EYQOr9fwde5+thWUsqOwjJ2F5ewsLGNnURk5+8rYe+DQ/Mset5CVEkf/tHgiI1zsKrLb795fTnWdqTljI930TY2jX1o8/VLj6JEYTV5xBTsKy+xjn32uqPYe/ExqfBSDeyQwKD2Bgd0T2JRXwttrc9laUIoIjOuTzOnHpFNZ7eWZZVvZUVhGepdoZk7M5OJxmaQlRFFeVcMX2/axbFMBSzcXsGp7IVU1Bo9bGJ3ZlRMGpjJpQCrDM5Jwu4Jb8unI1uQU8fSyLSxYvZPyKi9JsZ6D7U/RHhfDeyUxKjOJ4RlJdI31EBsVQWykm9hIN3GREcRGuYmK6DxtUeVVNWzOO8DGvBI27SkhZ18ZfVJiGdE7ieG9EukaF+mX4wQlEYiIG9gAnArkAJ8DlxpjvvLZ5npguDHmWhG5BDjPGHNxY/vVRNB5FJVWsSnf/vFvyjvAprwSNuWVUF1j6JEYTc+kGHokRjuPGKI8LrbkH2BT3gG+yz/A5nz7n6b2Tzg1PopeXWPISIqhZ1I0vZJiGNg9gaPTE0iNjzri+MYYvtldzNtrc3l7bS7rc+1d2BP7JXPFsVmcOqR7vaWUWqWV1WRv2ccnG/P5eGM+63buB6BLdAQT+6XQq2uMcwGLOHgRi4l04xKh2uulusbYZ6+husaehMftwuMWIiNczmsXEW4Bc6hB37dxv6rGS2W11z47ryuqvRSXV1NUZhv4C0urKCyroqi0ksgIFxldY+mdHENG11gyusaQ0TWGrrGRlFd5Ka+uobyq9uGlvKqGMt/XlfZ9RXUNiTEeUuKiSE2IIjUuktSEKFLiIvFEuPB6DV5jS1deY6jxGj7dVMDTy7ayenshMR4300f1YubETIb06MLOonJWbtvHF1sL+WLbPtbtLKKqpuFrU2p8FFkpsfRJibPPqXH0SY4lJT6SGI/9zqMiXLg6SEIur6ph+95SthaUsm2vfWwpsH/zvn/DIvbc8oorDn42MzmW4RmJjMhI4viBqQzu0aVVMQQrERwL3GmMOd15/ysAY8w9Ptu842yzVEQigFwgzTQSlCYC5au8qoa84grSEqKI9rTtV+LWggPUeA39WnlvRUFJBZ9uKuCTjfks3VzA3pJKDlRW4w1C7avHLSTGRJIU6yEpxkNSrIcuMR4qqr3k7Ctjx75S8ksqm95RHS6xnQI8ES6Ky6sPdkFurv5pcVw2sQ/fH5PRaDtLeVUNm/JKKCmvprSyhgOV1ZRW2OeS8mp2FJaxpeAAWwtK2VVU3uB+oj0uG6/bZbtH43STdtaLCDVeQ40xTvKySeuwK5Ac9nSwi3XtM8jBLtguAZdLDnt9oKKa3fsr8BUX6aZPShz9u8XTPy2OAd3i6Z8WT9/UOKI9bvaXV7E2p4jVOUV8mVPIlzlF7Cgs4/rJ/fnF1EEt+s4PnkaQuo/2Arb7vM8BJjS0jTGmWkSKgBTgsPGURWQWMAsgM1OnZlSHRHvc9E6O9cu++qTEtenzKfFRnDOiJ+eM6HlwmTGGimovZc7FrKyyBq+BCLcQ4RIi3C48LjlYlVTtNQd/4VfVmIO/9F0iCBx+MROIdLuIjHAdLEFERriIdLuIinA12Y5SWlnNzsIytu8ro7C0kugIN9GRbqIj3MREuon2uIiOsFUyUR63c0GVg/v1eg2FZVUUlFSQV1JBfkklBSUV1HjNwQuh2yW4XIJbhKzUOCb0TW5Wh4Foj5uhPROb9b2XVdYc/IVdVFZ1sNRSVmlLNqWVNVTVeJ3S1KGLvAGMAbfLiVPk4HPt92wwzr/joeMZY+rtcm0MNqE4SaXGKcVFR7jpkxJLZnIsmSmx9EmOJTkustHvoUu0h+MGpHLcgNSDy3xLCf4WEi1cxpg5wBywJYIgh6NUs4kI0R430R633+p6/SU2MoIB3RIY0C2hVZ93uYRk56bDgd1btw9/iIl0c3S6rQLszHw7NPhbIJvndwC9fd5nOMvq3capGkrENhorpZRqJ4FMBJ8DA0Wkr4hEApcAC+psswC4wnl9AfBBY+0DSiml/C9gVUNOnf9PgXew3UcfN8asE5G7gGxjzALgMeBpEdkI7MUmC6WUUu0ooG0Expg3gTfrLPudz+ty4MJAxqCUUqpxegufUkqFOU0ESikV5jQRKKVUmNNEoJRSYS7kRh8VkTxgays/nkqdu5bDSLieu553eNHzblgfY0xafStCLhG0hYhkNzTWRmcXrueu5x1e9LxbR6uGlFIqzGkiUEqpMBduiWBOsAMIonA9dz3v8KLn3Qph1UaglFLqSOFWIlBKKVWHJgKllApzYZMIRGSqiHwjIhtFZHaw4wkUEXlcRPaIyFqfZcki8p6IfOs8dw1mjIEgIr1F5EMR+UpE1onIjc7yTn3uIhItIp+JyGrnvH/vLO8rIsudv/d5zlDwnY6IuEVkpYi87rzv9OctIltEZI2IrBKRbGdZm/7OwyIRiIgbeAA4AxgCXCoiQ4IbVcA8AUyts2w28L4xZiDwvvO+s6kGbjHGDAEmAj9x/o07+7lXAFOMMSOAkcBUEZkI/B/wD2PMAGAf8MPghRhQNwJf+7wPl/M+yRgz0ufegTb9nYdFIgDGAxuNMZuNMZXA88C5QY4pIIwxS7BzO/g6F3jSef0kML09Y2oPxphdxpgvnNfF2ItDLzr5uRurxHnrcR4GmAK86CzvdOcNICIZwFnAo857IQzOuwFt+jsPl0TQC9ju8z7HWRYuuhtjdjmvc4HuwQwm0EQkCxgFLCcMzt2pHlkF7AHeAzYBhcaYameTzvr3/k/gF4DXeZ9CeJy3Ad4VkRUiMstZ1qa/85CYvF75jzHGiEin7TMsIvHAS8BNxpj99kei1VnP3RhTA4wUkSTgFWBQcCMKPBE5G9hjjFkhIpODHE57O94Ys0NEugHvich635Wt+TsPlxLBDqC3z/sMZ1m42C0iPQCc5z1BjicgRMSDTQJzjTEvO4vD4twBjDGFwIfAsUCSiNT+0OuMf++TgGkisgVb1TsF+Bed/7wxxuxwnvdgE/942vh3Hi6J4HNgoNOjIBI7N/KCIMfUnhYAVzivrwBeC2IsAeHUDz8GfG2M+bvPqk597iKS5pQEEJEY4FRs+8iHwAXOZp3uvI0xvzLGZBhjsrD/nz8wxsygk5+3iMSJSELta+A0YC1t/DsPmzuLReRMbJ2iG3jcGHN3cCMKDBF5DpiMHZZ2N3AH8CowH8jEDuF9kTGmboNySBOR44GPgDUcqjP+NbadoNOeu4gMxzYOurE/7OYbY+4SkX7YX8rJwEpgpjGmIniRBo5TNXSrMebszn7ezvm94ryNAJ41xtwtIim04e88bBKBUkqp+oVL1ZBSSqkGaCJQSqkwp4lAKaXCnCYCpZQKc5oIlFIqzGkiUKodicjk2pEyleooNBEopVSY00SgVD1EZKYzzv8qEXnEGditRET+4Yz7/76IpDnbjhSRZSLypYi8UjsWvIgMEJGFzlwBX4hIf2f38SLyooisF5G54jsgklJBoIlAqTpEZDBwMTDJGDMSqAFmAHFAtjFmKLAYe9c2wFPAL40xw7F3Ntcunws84MwVcBxQOzrkKOAm7NwY/bDj5igVNDr6qFJHOhkYA3zu/FiPwQ7i5QXmOds8A7wsIolAkjFmsbP8SeAFZzyYXsaYVwCMMeUAzv4+M8bkOO9XAVnAxwE/K6UaoIlAqSMJ8KQx5leHLRT5bZ3tWjs+i+/YNzXo/0MVZFo1pNSR3gcucMZ7r50Ptg/2/0vtyJY/AD42xhQB+0TkBGf5ZcBiZ5a0HBGZ7uwjSkRi2/MklGou/SWiVB3GmK9E5HbsLFAuoAr4CXAAGO+s24NtRwA77O/DzoV+M3CVs/wy4BERucvZx4XteBpKNZuOPqpUM4lIiTEmPthxKOVvWjWklFJhTksESikV5rREoJRSYU4TgVJKhTlNBEopFeY0ESilVJjTRKCUUmHu/wGRX4JWWX4EWQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Training and Validation Losses',size = 12)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9e1599c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n"
     ]
    }
   ],
   "source": [
    "l = len(x_s_test)\n",
    "_, balance2 = divmod(l,batch_size)\n",
    "x_s_test = list(x_s_test)\n",
    "fill_number = batch_size - balance2\n",
    "for i in range(fill_number):\n",
    "    x_s_test.append(np.zeros(sentence_maxlen,))\n",
    "    print(len(x_s_test))\n",
    "x_s_test = np.array(x_s_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f925adf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "bccbccbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.2963    0.2162    0.2500        37\n",
      "           1     0.4368    0.5067    0.4691        75\n",
      "\n",
      "   micro avg     0.4035    0.4107    0.4071       112\n",
      "   macro avg     0.3665    0.3614    0.3596       112\n",
      "weighted avg     0.3904    0.4107    0.3967       112\n",
      " samples avg     0.2599    0.2599    0.2599       112\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "y_pred = np.round(model2.predict(x_s_test, batch_size = 16))\n",
    "print(classification_report(y_test, y_pred[:-fill_number], digits=4, labels = [0,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9bf70af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n",
    "# cvscores = []\n",
    "\n",
    "# for train, val in kfold.split(x_s, classes_[y_train.argmax(1)]): \n",
    "#     model2.fit(x_s[train], y_train[train], epochs = 5, batch_size = 16, verbose=1)\n",
    "# #     scores = model2.evaluate(x_s[val], y_train[val], verbose=0)\n",
    "# #     print(\"%s: %.2f%%\" % (model2.metrics_names[1], scores[1]*100))\n",
    "# #     cvscores.append(scores[1] * 100)\n",
    "# # print(\"%.2f%% (+/- %.2f%%)\" % (np.mean(cvscores), np.std(cvscores)))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccd4d929",
   "metadata": {},
   "outputs": [],
   "source": [
    "#embeddings_weights = get_embeddings('wikipedia',300,vocabulary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d3085bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Do we have to do this?\n",
    "# x_s_ = tf.convert_to_tensor(\n",
    "#     x_s_, dtype=tf.float32, dtype_hint=None, name=None\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dc757d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# i = 0\n",
    "# j = 16\n",
    "# while j<=640:\n",
    "#     inputs = x_t[i:j]\n",
    "#     lstm = LSTM(128, return_sequences=True, return_state=True)\n",
    "#     whole_seq_output, h_0, c_0 = lstm(inputs,initial_state = [h_0, c_0])\n",
    "#     i += 16\n",
    "#     j += 16\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "478352ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# inputs = Input(batch_shape=(16,1,18))\n",
    "# lstm = LSTM(128, return_sequences=True, return_state=True)\n",
    "# whole_seq_output, h_0, c_0 = lstm(inputs)\n",
    "# model = Model(inputs=inputs, outputs=whole_seq_output)\n",
    "# k = model.predict(x_t[:640])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1da6e126",
   "metadata": {},
   "outputs": [],
   "source": [
    "# final_memory_state = tf.reshape(\n",
    "#     final_memory_state, (16,1,128), name=None\n",
    "# )\n",
    "\n",
    "# final_carry_state = tf.reshape(\n",
    "#     final_carry_state, (16,1,128), name=None\n",
    "# )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e375ad97",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# inputs2 = Input(shape=(1,sentence_maxlen,), name = 'Input')\n",
    "# #embedded_inputs = Embedding(embeddings_weights.shape[0], embeddings_weights.shape[1], weights=[embeddings_weights], name = 'Embedding')(inputs2)\n",
    "# #embedded_inputs2 = Dropout(0.2)(embedded_inputs)\n",
    "# lstm, s1,s2 = LSTM(128,return_sequences=True,return_state=True,dropout=0.3,name = 'lstm')(inputs2, initial_state = i_states)\n",
    "# #lstm = LSTM(128,return_sequences=True,dropout=0.3,name = 'lstm')(inputs2)\n",
    "# flat = Flatten(name = 'Flatten')(lstm)\n",
    "# output = (Dense(3,activation='softmax',name = 'Dense'))(flat)\n",
    "# model2 = Model(inputs=inputs2, outputs=output)\n",
    "# model2.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])    \n",
    "# model2.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59727728",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_t =np.array( [x.reshape(1,18) for x in x_t[:640]])\n",
    "\n",
    "# x_t = tf.convert_to_tensor(\n",
    "#     x_t, dtype=tf.float32, dtype_hint=None, name=None\n",
    "# )\n",
    "\n",
    "# inputs_list = [tf.squeeze(x) for x in\n",
    "#                 tf.split(1, sentence_maxlen, embedded_inputs)]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlpKernel",
   "language": "python",
   "name": "nlpkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}