per_target_all_deep_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a47c95f5-c1ba-4a45-bf43-917fd3c30b38",
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "import re\n",
    "import os\n",
    "import nltk\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "SEED = 1013\n",
    "np.random.seed(SEED)\n",
    "#nltk.download('stopwords')\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.corpus import stopwords, twitter_samples \n",
    "from stance_utils import *\n",
    "from stance_models import * \n",
    "#from parameters import *\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras import Sequential\n",
    "#from tensorflow.keras.layers import Dropout,Concatenate,Dense, Embedding, SpatialDropout1D, Flatten, GRU, Bidirectional, Conv1D,MaxPooling1D\n",
    "\n",
    "from tensorflow.keras.layers import RNN, Dropout,Concatenate,Dense, Embedding,LSTMCell, LSTM, SpatialDropout1D, Flatten, GRU, Bidirectional, Conv1D, Input,MaxPooling1D\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras import Model\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "stemmer = PorterStemmer()\n",
    "tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)\n",
    "stopwords_english = stopwords.words('english')\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import keras.backend as K\n",
    "from keras.layers import Lambda\n",
    "import random\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ef1eb2ac-3c04-4226-ad5f-ae132b6fae72",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = {'FAVOR': np.array([1, 0, 0]), 'AGAINST': np.array([0, 1, 0]), 'NONE': np.array([0, 0, 1])}\n",
    "classes_ = np.array(['FAVOR', 'AGAINST', 'NONE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7eb99a04-3910-453c-9c3a-731e4ac5e91a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_file_m = '/data/parush/stance_mohammed/new_train.txt'\n",
    "test_data_file_m = '/data/parush/stance_mohammed/new_test.txt'\n",
    "TARGETS_m = [ 'Atheism','Climate Change is a Real Concern', 'Feminist Movement','Hillary Clinton', 'Legalization of Abortion', 'Donald Trump']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9c4c124e-fc22-45b5-b049-f7eec3501d7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "790\n"
     ]
    }
   ],
   "source": [
    "c = 0\n",
    "with open(test_data_file_m, 'r') as trainfile:\n",
    "    for line in trainfile: \n",
    "        line = line.replace('#SemST', '').strip()\n",
    "        line = line.split('\\t')\n",
    "        if line[0].strip() != 'ID' and line[3] == 'AGAINST':\n",
    "            c += 1\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "373aecfb-2e42-4d16-9e4b-a3e52f542dcb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "9f08aad6-d353-4f23-82d6-034c785aaff3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_test(train_file, test_file, target):\n",
    "    \n",
    "    \n",
    "    sentence_maxlen = 0\n",
    "    target_maxlen = 0\n",
    "    x_s_token = []\n",
    "    x_t_token = []\n",
    "    y_train = []\n",
    "\n",
    "    \n",
    "    with open(train_file, 'r') as trainfile:\n",
    "        \n",
    "        for line in trainfile: \n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "            \n",
    "            \n",
    "            if line[0].strip() != 'ID' and target in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_s_token.append(tweet)\n",
    "                target_ = line[1].strip().lower().split()\n",
    "                if len(target_) > target_maxlen:\n",
    "                    target_maxlen = len(target_)\n",
    "                x_t_token.append(target_)\n",
    "                y_train.append(classes[line[3].strip()])\n",
    "    c_len = len(x_s_token)\n",
    "    print(\"Current x_s_token len is: \", c_len)\n",
    "                \n",
    "#     with open('rich_data/dt_ath_hc/Donald Trump_Atheism_Hillary Clinton0.5.txt','r') as new_file:\n",
    "#         for line in new_file:\n",
    "#             line = line.replace('#SemST', '').strip()\n",
    "#             line = line.split('\\t')\n",
    "#             tweet = line[0]\n",
    "#             tweet = process_tweet(tweet)\n",
    "#             if len(tweet) > sentence_maxlen:\n",
    "#                 sentence_maxlen = len(tweet)\n",
    "#             x_s_token.append(tweet)\n",
    "#             x_t_token.append(target_)\n",
    "#             y_train.append(classes[line[1].strip()])\n",
    "#     print(\"Added {} more examples\".format(len(x_s_token)-c_len))\n",
    "#     print(\"Total tweet {} and labels {}\".format(len(x_s_token), len(y_train)))\n",
    "        \n",
    "    \n",
    "\n",
    "                \n",
    "    \n",
    "\n",
    "                               \n",
    "    \n",
    "    x_s_test_token = []\n",
    "    x_t_test_token = []\n",
    "    y_test = []\n",
    "    with open(test_file, 'r') as testfile:\n",
    "        for line in testfile:\n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "        \n",
    "\n",
    "            \n",
    "            if line[0] != 'ID' and target in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_s_test_token.append(tweet)\n",
    "                target_ = line[1].strip().lower().split()\n",
    "                if len(target_) > target_maxlen:\n",
    "                    target_maxlen = len(target_)\n",
    "                x_t_test_token.append(target_)\n",
    "                y_test.append(classes[line[3].strip()])\n",
    "\n",
    "\n",
    "    \n",
    "    return x_s_token, x_t_token, x_s_test_token, x_t_test_token, y_train, y_test, sentence_maxlen, target_maxlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd8ba098-4a9a-4a35-876b-4b64d8f4b8d2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "08ddd7e6-fbc1-4815-9486-7ec4fd80ddd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current x_s_token len is:  664\n",
      "Current x_s_token len is:  653\n",
      "Total words in vocab are 2349\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0.]\n",
      "X_S_token 664, y_train 664, y_test 280\n",
      "X_S 672, y_train 672, y_test 280\n",
      "(672, 18, 100)\n",
      "Model: \"bicond\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input (InputLayer)              [(None, 18)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Embedding (Embedding)           (None, 18, 100)      234900      Input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     (16, 60)             38640       Embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "back_lstm (LSTM)                (16, 60)             38640       Embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_20 (Concatenate)    (16, 120)            0           lstm[0][0]                       \n",
      "                                                                 back_lstm[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Flatten (Flatten)               (16, 120)            0           concatenate_20[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Dense (Dense)                   (16, 3)              363         Flatten[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 312,543\n",
      "Trainable params: 312,543\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Training bicond on Atheism\n",
      "Training bicond on Atheism\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.2500    0.2609    0.2553        46\n",
      "           1     0.6739    0.4921    0.5688       189\n",
      "\n",
      "   micro avg     0.5645    0.4468    0.4988       235\n",
      "   macro avg     0.4620    0.3765    0.4121       235\n",
      "weighted avg     0.5909    0.4468    0.5074       235\n",
      " samples avg     0.3750    0.3750    0.3750       235\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "for target in TARGETS_m:\n",
    "    \n",
    "\n",
    "    batch_size = 16\n",
    "    units = 60\n",
    "    opt = keras.optimizers.Adam(learning_rate=1e-3)\n",
    "    num_classes = 3\n",
    "    bicondORnot = True # Set True in case of using bicond model\n",
    "    \n",
    "    x_s_token, x_t_token, _, _, y_train, _, sentence_maxlen, target_maxlen  = train_and_test(train_data_file_m, test_data_file_m, TARGETS_m[2])\n",
    "    _, _, x_s_test_token, x_t_test_token, _, y_test, _, _  = train_and_test(train_data_file_m, test_data_file_m, TARGETS_m[4])\n",
    "    vocabulary = build_vocab(x_s_token + x_t_token )\n",
    "    vocab_size = len(vocabulary)\n",
    "    print(\"Total words in vocab are\",vocab_size)\n",
    "    embedding_matrix = get_embeddings('twitter',100,vocabulary)\n",
    "    print('X_S_token {}, y_train {}, y_test {}'.format(len(x_s_token), len(y_train), len(y_test)))\n",
    "    \n",
    "    if bicondORnot:\n",
    "        y_train = np.asarray(y_train)\n",
    "        _,balance = divmod(len(y_train),batch_size)\n",
    "        balance = batch_size-balance\n",
    "        y_train = list(y_train)\n",
    "        for i in range(balance):\n",
    "            index = np.random.randint(1, len(y_train))\n",
    "            x_s_token.append(x_s_token[index])\n",
    "            y_train.append(y_train[index])\n",
    "            x_t_token.append(x_t_token[index])\n",
    "            \n",
    "                \n",
    "                    \n",
    "     \n",
    "            \n",
    "            \n",
    "    x_s = [tweet_to_tensor(each_s,vocabulary) for each_s in x_s_token]\n",
    "    x_s = pad_sequences(x_s, maxlen = sentence_maxlen, padding = 'post')\n",
    "    x_s_test = [tweet_to_tensor(each_s,vocabulary) for each_s in x_s_test_token]\n",
    "    x_s_test = pad_sequences(x_s_test, maxlen = sentence_maxlen, padding = 'post')\n",
    "\n",
    "    x_t = [tweet_to_tensor(each_s,vocabulary) for each_s in x_t_token]\n",
    "    x_t = pad_sequences(x_t, maxlen = sentence_maxlen, padding = 'post')\n",
    "    x_t_test = [tweet_to_tensor(each_s,vocabulary) for each_s in x_t_test_token]\n",
    "    x_t_test = pad_sequences(x_t_test, maxlen = sentence_maxlen, padding = 'post')\n",
    "    shuffle_indices = np.random.permutation(np.arange(len(y_train)))\n",
    "    x_s = x_s[shuffle_indices]\n",
    "    x_t = x_t[shuffle_indices]\n",
    "    y_train = np.asarray(y_train)\n",
    "    y_train = y_train[shuffle_indices]\n",
    "    y_test = np.asarray(y_test)\n",
    "    \n",
    "    # Select the model from below......\n",
    "    \n",
    "    print('X_S {}, y_train {}, y_test {}'.format(len(x_s), len(y_train), len(y_test)))\n",
    "    #model = biLSTMCNN(embedding_matrix, num_classes, sentence_maxlen)\n",
    "    #model = biLSTM(embedding_matrix, num_classes)\n",
    "    model = bicond(units, opt, embedding_matrix, x_t, batch_size, sentence_maxlen,num_classes)\n",
    "    print('Training {} on {}'.format(model.name, target))\n",
    "    \n",
    "\n",
    "    \n",
    "    if model.name == 'bicond': # making test_set % batch_size = 0 and validation_split % batch_size = 0\n",
    "        v_num = len(y_train)//10 \n",
    "        _, b3 = divmod(v_num,16)\n",
    "        v_split = (v_num  + (batch_size-b3)) / len(y_train)\n",
    "        history = model.fit(x_s, y_train, epochs = 50, batch_size = batch_size,validation_split = v_split,  verbose=0)\n",
    "        l = len(x_s_test)\n",
    "        _, balance2 = divmod(l,batch_size)\n",
    "        x_s_test = list(x_s_test)\n",
    "        fill_number = batch_size - balance2\n",
    "        for i in range(fill_number):\n",
    "            x_s_test.append(np.zeros(sentence_maxlen,))\n",
    "            \n",
    "        x_s_test = np.array(x_s_test)\n",
    "        y_pred = np.round(model.predict(x_s_test, batch_size = 16))\n",
    "        print('Training {} on {}'.format(model.name, target))\n",
    "        print(classification_report(y_test, y_pred[:-fill_number], digits=4, labels = [0,1]))\n",
    "        break\n",
    "    else:\n",
    "        kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n",
    "        cvscores = []\n",
    "\n",
    "        for train, val in kfold.split(x_s, classes_[y_train.argmax(1)]):\n",
    "            history = model.fit(x_s[train], y_train[train], epochs = 50, batch_size = batch_size, verbose=0)\n",
    "            scores = model.evaluate(x_s[val], y_train[val], verbose=0)\n",
    "            print(\"%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))\n",
    "            cvscores.append(scores[1] * 100)\n",
    "        print(\"%.2f%% (+/- %.2f%%)\" % (np.mean(cvscores), np.std(cvscores)))\n",
    "\n",
    "        y_pred = np.round(model.predict(x_s_test))\n",
    "        print('Training {} on {}'.format(model.name, target))\n",
    "        print(classification_report(y_test, y_pred, digits=4, labels = [0,1]))\n",
    "    break\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "b5668309-53e2-466c-b413-0df0aac2eecf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{90: 39.42, 80: 48.35, 70: 48.53, 60: 46.37, 50: 45.8}"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{90:39.42, 80:48.35, 70:48.53, 60:46.37, 50:45.8}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "946d2dd4",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (1617502999.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"/tmp/ipykernel_42323/1617502999.py\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    {90: , 80:, 70:, 60:, 50:}\u001b[0m\n\u001b[0m         ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "{90: , 80:, 70:, 60:, 50:}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8215158-4421-4a4c-88c0-b60598dbf465",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0decbff1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b255d44e-796b-498e-9736-ecf96a732ff1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef1fa903",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2609d376",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cf8efe0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2da81e9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8242e563",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fce7612",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eabf7a4e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca0f84be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26fc23fb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "83413cc7",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'val_loss'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_32347/1991655455.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training and Validation Losses'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m12\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxlabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'epoch'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'val_loss'"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAx6ElEQVR4nO3deXyb5ZXo8d+x5X1NbGezs68EEhIICYFCW+hQloGwFphOSzu00Nty296WmYFOh/Yy7dzS6ZR2pkDLDEwpXSCl0KYtlH1LgBAnIXviOLuz2rEdW1IsWdK5f0iyHUe2ZUuytZzv55NPpFev5OdN5KOj8z7veURVMcYYk76yRnoAxhhjEssCvTHGpDkL9MYYk+Ys0BtjTJqzQG+MMWnOMdID6K2yslKnTJky0sMwxpiUsnbt2iZVrYr0WNIF+ilTplBbWzvSwzDGmJQiIvv6esxKN8YYk+Ys0BtjTJqzQG+MMWnOAr0xxqQ5C/TGGJPmLNAbY0yas0BvjDFpzgJ9BtpxpJ339zSP9DCMMcPEAn0G+tErdXzjuU0jPQxjzDCxQJ+B2jt8tJ3sHOlhGGOGiQX6DOT0+HB5fCM9DGPMMLFAn4HcXh8ur59AwJaRNCYTWKDPQC6PP/i317J6YzKBBfoM5A4F+HDAN8akNwv0GSgc4J1WpzcmI1igzzBeXwCvPwBgJ2SNyRAW6DPMSW93ucYCvTGZIapALyKXi8gOEakXkXsiPJ4nIk+HHl8tIlNC26eIyEkR+SD056dxHr8ZpJ4nYK10Y0xmGHApQRHJBh4C/gpoANaIyApV3dpjt9uBFlWdISK3AA8AN4ce26WqC+I7bDNUPbN4C/TGZIZoMvrFQL2q7lZVL/AUsKzXPsuAJ0K3nwEuFRGJ3zBNvLisdGNMxokm0FcDB3rcbwhti7iPqvqAE0BF6LGpIrJeRN4UkYtiHK+JkfuUjN6mVxqTCQYs3cToMDBJVY+LyLnA70XkTFVt67mTiNwB3AEwadKkBA8ps/Us11hGb0xmiCajPwhM7HG/JrQt4j4i4gDKgOOq6lHV4wCquhbYBczq/QNU9VFVXaSqi6qqqgZ/FCZq7h6lG6vRG5MZogn0a4CZIjJVRHKBW4AVvfZZAdwWun0j8JqqqohUhU7mIiLTgJnA7vgM3QxFeNaNI0ss0BuTIQYs3aiqT0TuAl4EsoHHVXWLiNwP1KrqCuAx4EkRqQeaCX4YAFwM3C8inUAA+IKq2ooXI8gdqstXleRZ6caYDBFVjV5Vnwee77Xtvh63O4CbIjzvd8DvYhyjiaNwFl9VkmcZvTEZwq6MzTBur4+CnGxK8h2W0RuTISzQZxiX109RXjbFeQ7rXmlMhrBAn2HcHh+FuQ6K8hxWujEmQ1igzzBOj5+iPAfFFuiNyRgW6DOM2+ujKDeborxgjV7VlhM0Jt1ZoM8wLq+fwlBG7wsoHl9gpIdkjEkwC/QZxu0JZvTFecGZtTbzxpj0Z4E+w7g8Porygidjg/dt5o0x6c4CfYZxef2hjD4bgHZP5wiPyBiTaBboM4zb66PQMnpjMooF+gzi9QXo9GvXrBuwGr0xmcACfQYJB/WiPAcloUBvc+mNSX8W6DNIuEVxUa7DMnpjMogF+gwSXnSkMK+7dGMZvTHpzwJ9Bukq3eQ6KMoNzrqxQG9M+rNAn0HCM2yK8hw4srPIz8my0o0xGcACfQYJ1+gLQ9l8sLGZTa80Jt1ZoM8gbm/3rBsg1JPeMnpj0p0F+gzSVboJZfRFFuiNyQgW6DNIz3n04b/bLdAbk/Ys0GcQV2h6ZUFOd43eMnpj0p8F+gwSXEYwm6wsAax0Y0ymsECfQVxeP4W5jq77NuvGmMxggT6DuDy+rvbEAMV52ZbRG5MBLNBnELfXd0pGX5Tn4GSnH3/A1o01Jp1ZoM8gLo+folMyeut3Y0wmsECfQSJl9GAdLI1JdxboM4jT4+vK4sECvTGZwgJ9BnF7/V19bgBbfMSYDGGBPoO4PL6uLB6wdWONyRBRBXoRuVxEdohIvYjcE+HxPBF5OvT4ahGZ0uvxSSLiFJG74zRuM0iqitt76snY8G2np3OkhmWMGQYDBnoRyQYeAq4A5gK3isjcXrvdDrSo6gzgQeCBXo//EHgh9uGaofL4AvgCetoFU4BdNGVMmosmo18M1KvqblX1Ak8By3rtswx4InT7GeBSEREAEbkW2ANsicuIzZCElxEsyu2Z0dvJWGMyQTSBvho40ON+Q2hbxH1U1QecACpEpBj4R+D/9vcDROQOEakVkdrGxsZox24GIRzMC/MiZfQW6I1JZ4k+Gftt4EFVdfa3k6o+qqqLVHVRVVVVgoeUmcIZfc/plXmOLBxZYhm9MWnOMfAuHAQm9rhfE9oWaZ8GEXEAZcBxYAlwo4h8HygHAiLSoao/iXXgZnDCWXvP6ZUiYh0sjckA0QT6NcBMEZlKMKDfAvxNr31WALcB7wI3Aq+pqgIXhXcQkW8DTgvyI6P3MoJhxbb4iDFpb8BAr6o+EbkLeBHIBh5X1S0icj9Qq6orgMeAJ0WkHmgm+GFgkkh4rnzPjB6CUywtozcmvUWT0aOqzwPP99p2X4/bHcBNA7zGt4cwPhMn4Yy+uFdGHyzd2PRKY9KZXRmbIbpm3eSeXrqxWTfGpDcL9BkivF5szytjwdaNNSYTWKDPEG6PD5HuhcHDiiyjNybtWaDPEC6vn6JcB6ELlrtY6caY9GeBPkO4PL7TZtxA96yb4GxYY0w6skCfIVxe/2lz6AGK83IIKHR0BkZgVMaY4WCBPkO4+8joi7taFVv5xph0ZYE+Q7i8vogZvXWwNCb9WaDPEC6P/5QWxWFF1sHSmLRngT5DuLy+U1oUh1mrYmPSnwX6DOEeIKO30o0x6csCfYboq0ZvGb0x6c8CfQZQVVweH0W5fQd6a2xmTPqyQJ8BPL4AAYXCvMgXTAE4PZ3DPSxjzDCxQJ8BwvX3SBl9eJvTMnpj0pYF+gzg7upceXqgz8oSCnNt8RFj0pkF+gzg7MroTy/dgLUqNibdWaDPAOHVpSLNowfrYGlMurNAnwHCM2r6yuiLLKM3Jq1ZoM8A4Yw+Uo0+uD3bMnpj0pgF+gzg7Mro+yvd2KwbY9KVBfoM0F2jt9KNMZnIAn0GcEWR0VugNyZ9WaDPAG6vjyyB/JzI/90268aY9GaBPgM4Q31uei8MHlaU58DjC9Dpt+UEjUlHFugzgNvj77M+D9aq2Jh0Z4E+A7i8kTtXhtm6scakNwv0GcDt9fc5hx6gOC8HsFbFxqQrC/QZwOnxUdjHVbHQs1WxZfTGpKOoAr2IXC4iO0SkXkTuifB4nog8HXp8tYhMCW1fLCIfhP5sEJHr4jx+EwV3H6tLhRVbjd6YtDZgoBeRbOAh4ApgLnCriMzttdvtQIuqzgAeBB4Ibd8MLFLVBcDlwM9EpO+Ik6Ja3V4u/N5rbDjQOtJDicjt8Q+Q0dtygsaks2gy+sVAvaruVlUv8BSwrNc+y4AnQrefAS4VEVFVt6qGo0c+oPEYdLLZ0+TiYOtJth5uG+mhROTy+rqy9khs3Vhj0ls0gb4aONDjfkNoW8R9QoH9BFABICJLRGQLsAn4Qo/AnzZa3cFl+NpOJudyfC6Pn8J+Z91Y6caYdJbwk7GqulpVzwTOA+4Vkfze+4jIHSJSKyK1jY2NiR5S3DW7vAC0dyRfoFTV4PRKm0dvTMaKJtAfBCb2uF8T2hZxn1ANvgw43nMHVd0GOIGzev8AVX1UVRep6qKqqqroR58kWtzhQJ98GX1HZwBV+s3ocx1Z5GZn9dnBUlVZu68F1bSsvBmT9qIJ9GuAmSIyVURygVuAFb32WQHcFrp9I/CaqmroOQ4AEZkMzAH2xmXkSSQc6NuSMKN3hTpXFveT0UO4J33kD6pV9ce54ZF3WFV/POLjxpjkNmCgD9XU7wJeBLYBy1V1i4jcLyLXhHZ7DKgQkXrga0B4CuaHgA0i8gHwHPBFVW2K8zGMuGZXMEAmY0YfLsf0l9FDuFVx5Ix+zd5mANbvb4nv4IwxwyKqqY6q+jzwfK9t9/W43QHcFOF5TwJPxjjGpNeazBl9uEXxABl9fx0s14emjW48eCKuYzPGDA+7MjYOwidjk3HWzUDLCIb11ZM+EFA+CGXymxos0BuTiizQx0F4emUyzrpxDqp0c/r4dze5aOvwMWdcCUfaOjjW3pGQcRpjEscCfRw0J/GsG7c3ttJNuC5/2wVTANhs5RtjUo4F+hipKi3hefQeH4FAck1BDGfp/bUphvCsmwiB/kArpfkO/nr+eERgU0NyXv1rjOmbBfoYOT0+fAFlTEkeqt3TGZNFd0Y/tFk36/e3smDSKEryc5heVcymg62JGKYxJoEs0MeoJTS1cnJFIZB8dfruGn3/pZuSPAcur++Ui6JcHh87jrSxcGI5APOqy9hoJ2SNSTkW6GMUvlhq0ugiANqSrE7v9vrIzhLyHP3/VxflOVDt/gYAsLHhBAGFhZPKgWCgP9bu4WibnZA1JpVYoI9R+ERssmb0rlCL4r4WBg+L1O9m/YHgidgFoYx+fk0ZYNMsjUk1Fuhj1Nor0CfbXHr3AC2Kw8L7tPcM9PtbmVZZRHlhLgBzJ5SSJbDJZt4Yk1Is0Mco3P5g4ujkzugH0jujV9XQidjyrn0Kcx3MGFNsgd6YFGOBPkYtLi9ZAjWjCoDkm0vvGmAZwbDe68Y2tJykyelh4aRRp+x3VnUZmw6esE6WxqQQC/QxanF7KS/MpawgB0i+fjcDLSMYVpIXHH94imW4v014xk3Y/OoyGts9HG3zxHWcxpjEsUAfoxa3l1GFOeQ5ssl1ZCXdrJuBlhEMC2f04dLN+v0t5OdkMWdcySn7zaspB2BjQ2tcx2mMSRwL9DFqcXUyKnSysjQ/Jwlr9L4B+9zA6evGrt/fyvyachzZp75F5o4PnpC1VgjGpA4L9DFqcXsZVRQO9I6km3Xj8voH7HMD3SdjnR4fHp+frYfauubP91SQm82ssSXWstiYFGKBPkbNrmDpBqAk35F0Gb07yow+ONc++A1gy6E2vP4ACyeOirjvvOoyNtsJWWNShgX6GKgqre7O7oy+ICepZt0EAoq70x/VrBsRoTg32MFy/f5WgIgZPcC8mjKanF4On7ArZI1JBRboY+Dy+vH6A4wO1ehL8h1JNevmZKcfVSiKYtYNdPekX7+/heryAsaW5kfcb1518ApZ63tjTGqwQB+DcHvi8MnYkrzkyujDnTQLo8joITjzxuXxn3ahVG9njC8lO0vshKwxKcICfQzCDc26SzfJVaN3h9eLjTKjL85zsLvJxcHWk6fNn+8pP8dOyBqTSizQx6AltIRg98nYHNxeP53+wEgOq4sryvViw4ryHGw7HFxYpK/6fNh8OyFrTMqwQB+DrtJNUXeNHsCZJFm9qyujjz7QA+RkC2dOKOt337Nqymh2eTnYejK2QRpjEs4CfQyaQ4F+dI8LpiB5Gpt11+ijK92UhAL93PGl5Of0/5z51day2JhUYYE+Bq1uLyLBaZXQndEnSxsE9xAz+t6NzCKZPa4ER5ZYJ0tjUoAF+hg0u72UF+SQnRVc1KMkP9zYLDkCfXeNPvrplTBwfR6CJ2RnjyuxQG9MCrBAH4MWd3efGwjOuoEkKt2E+tZEm9GHv5Es6GfGTU/za6xlsTGpILoIYCJqcXX3uYHuGn2y9LsJr/8abY3+2oXVlBbkMCm0iMpAzqou4zfvH6Ch5WTXwivGmORjGX0Mgn1uTg/0yZTRO7KE3Ozo/purywv41PmTB1xfNmx+dTlgSwsak+ws0Meg1d3ZNYceoDjZTsZ6g31uog3cgzVrXDEAO486E/L6xpj4sEA/RKpKs9vL6B6lm+wsoSg3O2kyeqfHF/VVsUOR58hmVGEOTU5bbcqYZBZVoBeRy0Vkh4jUi8g9ER7PE5GnQ4+vFpEpoe1/JSJrRWRT6O9L4jz+EXOy04/XF6C8R+kGkquDpdvri7rPzVBVleTR2G6B3phkNmCgF5Fs4CHgCmAucKuIzO212+1Ai6rOAB4EHghtbwKuVtV5wG3Ak/Ea+EjruliqKOeU7SX5DtpOJkdG7/L4E5rRQyjQW0ZvTFKLJqNfDNSr6m5V9QJPAct67bMMeCJ0+xngUhERVV2vqodC27cABSKSF4+Bj7TWrj43p2b0Jfk5tHuSJ6OPts/NUFUWW0ZvTLKLJtBXAwd63G8IbYu4j6r6gBNARa99bgDWqeppUUFE7hCRWhGpbWxsjHbsI6q5V5+bsNIkWmXK6fFHtbpULKos0BuT9IblZKyInEmwnHNnpMdV9VFVXaSqi6qqqoZjSDHralEcIaNPhnn0To+P405P1FfFDlVVSR4nO/1dF2cZY5JPNIH+IDCxx/2a0LaI+4iIAygDjofu1wDPAZ9W1V2xDjhZdC86cnqNfqQz+vf3NHPFj9+iyenhkjljEvqzqkqClTjL6o1JXtEE+jXATBGZKiK5wC3Ail77rCB4shXgRuA1VVURKQf+DNyjqqviNOak0OzuRATKCk4N9MFZN74RaQvg8fn5fy9s4+ZH30UQlt+5lGULelfZ4qsr0NsJWWOS1oAFXFX1ichdwItANvC4qm4RkfuBWlVdATwGPCki9UAzwQ8DgLuAGcB9InJfaNtlqnos3gcy3FrdXsoKcnD0uuq0JN+B1x/A4wsM2Oo3nrYfaeOrT33A9iPt3Lp4Iv901VyKE3wiFoInY8EyemOSWVSRQFWfB57vte2+Hrc7gJsiPO87wHdiHGNS6t3+IKxnB8vhCvSPr9zD917YTmlBDo/dtohLzxg7LD8XrHRjTCqwpmZD1Lv9QVhpfncHyzEliR/HgWY39/9pKx+ZXcW/33Q2FcXDO3t1VGEu2VliV8cak8SsBcIQ9ZXRD3cHy+1H2gH48qUzhz3IQ7DtQ0VRrmX0xiQxC/RD1OL2njaHHrp7ug/XzJu6o8FAP3NM8bD8vEjsoiljkpsF+iFq6dXQLCy8rOBwBfodR9qpLi/oOjcwEqwNgjHJzQL9EJz0+unoDFAeoUY/3OvG7jjSzuxxw3AyoB9VJXk0WUZvTNKyQD8E4atiR/cz62Y4Olh6fQF2NTqTItA3Oj22pKAxScoC/RCE+9z0blEMUJSbTZYwLB0s9x534Qsos8eOcKAvzqPTr5xIgtYPxpjTWaAfgq6MPkKNXkSCHSyHIaPfEZpxM2uEA32lzaU3JqlZoB+CllCL4t696MOGq99N3dF2srOEaVVFCf9Z/amyq2ONSWoW6IegpZ/SDQTn0g/HydgdR9qZUlE4rK0WIrF+N8YkNwv0QxAu3ZQX9J3Rtw1DRr/jaDtzxpUm/OcMxNogGJPcLNAPQYvLS2m+47SGZmHBGn1iA73b62N/s3vE6/MQbPuQ68iyjN6YJGWBfgha3J0RT8SGlRY4Et4Cof6YE1WYPW7krogNExFbacqYJGaBfgj6an8QVjoMs26SZcZNWGWJBXpjkpUF+iHoq6FZWEm+A6fHRyCQuAuI6o62k+fIYnLFyM64CasqzqPJ6R3pYRhjIrBAPwTBFsX9Z/QBBZc3cXX67UfamTm2mOwsSdjPGIwqy+iNSVoW6IcgmNH33URsODpY1h1tT5qyDUBVcS7NLg/+BH6LMcYMjQX6Qero9HOy099vjb67301iAn2r28vRNs+Itz7oqaokj4DCcZdl9cYkGwv0g9Rf+4Ow0oLEdrCsO+oEYNYINzPryebSG5O8LNAPUosrGLz7L90ktoPljtBiI3OSMNDbCVljko8F+kEKZ/T9n4xNbI1+x5E2SvIdjCvNT8jrD0VVcXAsltEbk3ws0A9SuEVxNDX6RF00VXfEyeyxJYgkx4wbgMqS4L+HBXpjko8F+kFqjSKj715lKv4Zvaqy42h7UtXnAQpzHRTlZlugNyYJWaAfpOZQjT7SMoJh+TnZ5DqyElK6Odbu4cTJzqSacRNma8cak5ws0A9Si9tLSb6DnD4amoWV5jsSMusm3PpgpJcPjMTWjjUmOVmgH6QWt7ffqZVhiepgWXc0uXrc9GQZvTHJyQL9IDW7vH0uONJTaX5iOlhuP9JOVUleVB82w63SOlimjd2NTlyexK+pYIaHBfpBanV3Mrqf+nxYotaNrTvanpT1eQg2NjtxshOPzz/SQzExcHp8XPHjt7n6JyvZ2+Qa6eGYOLBAP0jNrv5bFIclYt3YQECTrsdNT+GLpo7bRVMpbfvhNjy+APuOu1n20Cre2dU00kMyMbJAP0it7v5bFIclYt3YAy1uOjoDSXVFbE/WBiE9bDvcBsAvb19CVUken37sfX7z/v4RHpWJRVSBXkQuF5EdIlIvIvdEeDxPRJ4OPb5aRKaEtleIyOsi4hSRn8R57MPO4/Pj8vqjPBkb/4y+a7ERC/QmgbYebqesIIfzp43m2S9ewAUzKrn32U3c/8et1p00RQ0Y6EUkG3gIuAKYC9wqInN77XY70KKqM4AHgQdC2zuAfwbujtuIR1Cre+A59GEl+Tm4vX58/kDcfn440M8cM/LLB0ZSWRwK9DbzJqVtO9zGGeODV16X5ufw+G2L+OyFU3h81R5uf2JNwldPM/EXTUa/GKhX1d2q6gWeApb12mcZ8ETo9jPApSIiqupS1ZUEA37KC7c/GB1N6aYg/v1udhxtZ+LoAoryHHF7zXiqKLY2CKnOH1C2H2lj7viyrm2O7Cy+dfWZfPe6s1i5s4l/em7zCI7QDEU0gb4aONDjfkNoW8R9VNUHnAAqoh2EiNwhIrUiUtvY2Bjt04ZdSxR9bsIS0ZM+mWfcAOQ5sikvzKHJMvqUtfe4i47OAGeMP/199sklk7n+nGre2HHMSjgpJilOxqrqo6q6SFUXVVVVjfRw+tQcRZ+bsO5+N/H5muv1Bdjd6ErKK2J7qrK59CktfCL2jPGlER+/YHolbR2+rv1Maogm0B8EJva4XxPaFnEfEXEAZcDxeAwwmazf30quI4tJowsH3Lc03MEyToF+T5MLX0CTdmplmF00ldq2HmrDkSXMHBv5PNDS6cEv6u/uSrtf77QWTaBfA8wUkakikgvcAqzotc8K4LbQ7RuB11R1WL/bbT3UxnUPr6L+mDNhP+PNukaWTB1NQW72gPvGe93YDQ2tAMztI9NKFtYGIbVtO9zGjDHF5Dkiv8fHluYzrbKId3dboE8lAwb6UM39LuBFYBuwXFW3iMj9InJNaLfHgAoRqQe+BnRNwRSRvcAPgc+ISEOEGTtxUVmSy8aGEzyztiERL8/B1pPUH3Py4VnRlZZK41yjf6e+icriPGYk6YybsKoSy+hT2bbD7X2WbcLOn17B+3ua4zqjzCRWVDV6VX1eVWep6nRV/W5o232quiJ0u0NVb1LVGaq6WFV393juFFUdrarFqlqjqlsTcSBjSvL56OwqfreuISFvwLfqgieJL4420IfXjY1DvxtVZWX9cS6cUZFUi41EUlWSh9vrtz4pKajZ5eVIW0fEE7E9LZ1WgdPjY/Mhq9OniqQ4GRsvNy2aSGO7hzfr4j9z5626RsaX5Uc9h704L36lm7qjTpqcHi6cURnzayVaVXF47VjL6lNN+ARrz6mVkZw/LVint9YIqSOtAv0lc8ZQWZzL8toDA+88CD5/gJX1TXx4VlXUGbUjO4vC3Oy4XFyysj74C5UKgb7Sro5NWd0zbvrP6KtK8pg1tthOyKaQtAr0OdlZXLewmle3HeN4HDPKDw600t7hi7psExavfjer6puYVllEdXlBzK+VaOGM3gJ96tl6qI0xJXlUhP4P+7N0WgW1e1vw+qxOnwrSKtBDsHzjCyjPre89A3To3qxrJDtLBp1Rx6PfTac/wHu7j6dENg89+t1Y6SblbD3cxtwJ0c3qWjq9gpOdfjaGZoOZ5JZ2gX7W2BLOnljO8toDxGuG55t1jSyYWE5ZwcA9bnoqicNygh8caMXt9adMoB9dlEuWYEsKphivL8CuRueAM27ClkytQMTm06eKtAv0AJ9YVEPdUScbG07E/FrHnR42HTwR9bTKnkoLYl9OcOXOJrIk+FU5FWRnCRXFNpc+1dQfc9Lp16gD/aiiXOaMK7X59CkiLQP91WdPIM+RFZeTsivrm1BlSIE+HuvGrqpvYl5NOWVRdMxMFnZ1bOrZ2jXjJvoL8i6YXkHtvhY6Om1FsWSXloG+ND+HK+eNZ8WGQzG/Cd+sa2RUYQ5nVfc/5SySkhjXjW3v6GT9gVYunJ4a2XyYXTSVerYdbiM/J4uplUVRP2fptAq8vgDr97cmbmAmLtIy0APctKiG9g4fL245MuTXCASUt+qauGhmFdlZg79QqTTGjP79Pc34A8qHUqQ+H2aNzVLPtsNtzB5bMqj3+eJpo8kSrHyTAtI20J8/tYKJowtiKt9sO9JGk9Mz6GmVYSX5Drz+wJC/VaysbyLPkcU5k0cN6fkjpaokjyanN24nw0eKqrJ2X3PaL3auqmw93BZ1fT6sND/4Tfc9OyGb9NI20GdlCTeeM5FV9cc50Owe0muEr7C9eObQMurSGFsVr6pvYvHU0eTnDNxELZlUleTh9QdoO5nabRCe33SEGx55l2+m+UIbR9o6aHV3Rj21sqel0ypYf6CFk970/jBMdWkb6AFuOLcaEYbc6Oytukbmji9lTGn+kJ5fWjD0xmbH2jqoO+pMmWmVPVWGV5pypu7CYk6Pj/v/tIX8nCx+u7aBFzYdHukhJcxAPej7s3R6BZ1+pXZfc7yHZeIorQN9zahCPjSjkmfWNhAY5Io4To+P2r0tQy7bQGytileF+oikWn0eui+aOhaHOr0/oLy+/Rhu7/B+O/jxK3UcbfPwi79bwvyaMu59bhNH21L3g6s/W0PNyeYMYVGb86aMxpElNp8+DhLZDTStAz0Er5Q92HqSh9+oZ+2+5q7lAAfy7q7j+AI6pGmVYeHlBIcy82ZV/XHKC3OSvv98JGPi1O8mEFC+8ewmPvvzNVzzk1VsPzI83RJ3HGnn8VV7uXXxRBZPHc2DNy+go9PP3b/dMOiEIRVsO9zOpNGFXe/XwSjKczC/psxOyMYoEFDufHIt33the0JeP+0D/WVzxzK9qogfvFTHDY+8y8J/eZmF97/EDY+8w9//dgO/X3+QzgifpG/WHaMoN5tzYzgROtSe9KrKqvomLpxeSdYQZvuMtKriYKmryRndh2okqsq3Vmzh6doD3HhuDSdOdnLNT1bx5Hv7EnqSV1X5599vpjTfwT98fA4A06uK+eZVc3l7ZxM/f2fvoF+z2eVl+ZoD/OzNXUm51uq2w20DNjLrz9LpFWxsOIHTWlN3aXV7eej1+qgTyx+/upNXtx+junxoZeKBOBLyqkkkPyebF796MQ0tJ9nd5GR3o4tdjS52Nzp5fccxfru2gX97cQd3XDyNTyyaSEFuNqrKm3WNLJ1eSa5j6J+F3aWb0zN6l8eHI1siruSzu8nF4RMdKVmfh2Av/tzsrCFn9KrKd/68jSff28edF0/jnivmcNzl5evLN/DPv9/Myp2NfP+GsxNyEdmz6w7y/t5mHrhh3imLwH9yySRe336M7/1lOxfOqBxw7d4jJzp4aesR/rL5CKtD02QBdh5z8v0b5g/5A9zrC/D3z2xgSkURX/3YzJjXJ3B7few57uKaBROG/BpLp1Xy0Ou7WLO3mY/OHhPTeNKBqvL15Rt4dfsxXtp6lF99bklX2/JIXt56lB+/upMbz63hb8+fnJAxpX2gh2DL4CmVRUypLOKSOd3bVZXXth/j4Td28a0VW/iPV3fy2Qun8KGZVRxoPskdF0+P6eeGT8a2dXQSCCibD53grbpG3trZxLp9LZQX5vCda+dx+VnjTnneqq62xKl1oVSYiFBZnMufNh5iT5MTQULbg3/GlxXwySWTmFZ1em9/VeXfXtzBYyv38JkLpnDPFXNCr5fH/3zmPP575W6+/5cdXPkfb/PjWxawaMrouI37hLuTf31+G+dMKuemcyee8piI8MCN87n8R2/xlafW84e7LjztQ/pAs5sXNh/mL5uPsC50EdGMMcX8rw9P5/KzxvHKtqP86JWd5GQL37123qCDvapy77Ob+MMHh4Dg+Yu7Pz576AcMbD/SjmpsS1SeO3kUudlZvLfreFIF+vX7W/jpm7v40kdnML+mfNh+7uOr9vLq9mNcu2ACf9x4mM89sYaff3ZxxNlzuxqdfO3pD5hXXcZ3rj0rYQsLZUSg74uIcOkZY7n0jLGs2dvMw6/X84OX6vj3l+sA+PDModfnAYpys8kS+NXq/Tzyxi5a3MHM/qzqUm6/aCordzbxhV+u5ZqzJ/Dta85kdCiDXLmziZpRBVEtQp6srj+nhle2HWXfcTeqoGjob3hl6zEeW7mHj86u4rMXTuWimZVdb/D/fK2eh9/Yxa2LJ/Ktq+ee8sbPyhLuuHg6i6dW8OXfrOfmR9/jhnOqufm8SZwzqTzmX5IfvLSDFreXX9y+OGIQrizO44Eb5nP7E7X8+0t1fOPKM9h/3M3zmw/z/KbDXb2VzpxQyt2XzeLys8YxY0x35n/mhFJ8fuUnr9eTnSX8y7LB/WI/9Ho9v1vXwFcuncmxdg8/eb2e/Jws7rpk5pCPOZYZN2EFudksmFTOnzYeZnJFERdMr2ByRWFcg1ZDi5sHX95JYW42/3TVGQNOOX5/TzOf/Z/3cXn9vLrtGF/92Ez+10dmDOnCx8HY2NDK917Yxl/NHcuDNy/gI7PH8H+Wf8CXfrWOn37qXHKyuysETo+PO59cS44ji59+6tyETqPO6EDf03lTRvM/n13M1kNt/OytYC11UkVsgVZEmDW2hOMuLx+dM4aLZ1bxoZmVVIb6fXdeFuCRN3bxn6/t5J1dTXzn2nl87IwxvLv7OFfNG5/0ywb25+6Pz+4z22xs9/Cr1fv45Xv7+fTj7zNjTDGfuWAKJ0528sOX67h+YTXfvXZen8e/YGI5f/7yh3jgL9t5dt1Bltc2MGNMMZ9YVMN1C2u6Zv0MxsaGVn65eh+3LZ3CmRP6bndx6Rlj+eSSSTz61m7eqmtk+5F2AM6uKePeK+ZwxVnj+3zfiAhfv2wWnYEAP3tzN46srNM+zPryxw2H+MFLdVy3sJqvfmwmquDp9PODl+rIz8nmcxdN6/O5a/c1s35/K1efPYGxvaYKbzvcRkm+g5pRsa118IUPT+PeZzfxjec2AVBdXsDS6RVcML2CC2dUnvZzo9XW0cnDr+/i8VV7gGDpavOhE/zsb8/tc9rzO7uauP3ntYwvz+enf3su//HqTn7wUh1v7GjkwZsXMDFBCVR7Ryf/+zfrqSrO499unI+IcO3CapweH9/8/Wa+vnwDD968gOwsQVX5+99uYE+TiydvX5zwtSYk2a5eXLRokdbW1o70MOImENBQyaLvX+Zth9u4+7cb2HKojSVTR7N6TzP/eetCrj576HXTVODx+fnzxsM8vmoPmw8GM8u/nj+eH928AEd2dOdGnB4ff954iKfXHGDd/lYcWcIlc8bwN0smRb0imD+gXP/wKg6d6ODVr3+46yR6X9xeH5/879WowlXzxnPFvHHUjIo+eITPQTy2cg+fv2gq37jyjH7HuXZfC7f+13ucXVPGLz+3pKtk5PMH+MpTH/DnTYf5l2Vn8qmlU0553rr9LTz4ch1v7wyWAnOyhWULqrnj4mnMGhv8pnH9w6twZGex/M6lUY+/v+Pa3eTinV3Heae+iXd3H6fV3Ul2lnDnxdP48qUzo85aO/0Bnnp/Pw++spNml5frF1Zz98dns+FAK19bvoGyghwe/fS5p5Vk3qpr5PO/qGVyRSG/+tz5VJXkoar84YND/PPvN6PAt685kxvOqY5rIqWqfPmpD3h+02GeuuN8zutVUnzkjV088Jft3Lp4Ev963Vk88uYuvv+XHXzzqjP6/ZAeDBFZq6qLIj5mgT45dPq7s/tOv7L2mx+LaqWfdKCq1O5rYWPDCT69dPIpX28Ho/5YO8trG3h2XQNNTi+zxhbzuYumsWzBhIgnvU+4O3l2fQO/Xr2fncec/PiWBSxbUB3r4URFVfn2ii088e4+7vzwNP7Px2ZFDIL7j7u57uFVFOc7eO6LF3aV98K8vgBf/NVaXtl2jO/fOJ9PLJrIBwdaefDlOt6sa2R0US53XjyNj8wew69X72N5bQMnO/18ZHYVd1w0jc//opabFk3k29ecGfdjDASUbUfa+J9Ve3lmbQPTqop44Ib5pwXBnjr9AV7ddpR/e3EHuxpdnD9tNN+8au4pTQW3Hmrj87+opcnp4fs3zu/6P3tt+1G+8OQ6po8p5pe3Lz7t96ehxc3Xlm/g/T3NXDlvHH934VTOqi6LS8nk6TX7+cffbeLuy2b1WUp74C/beeSNXVw2dyyvbDvKVfMn8B+3LIjbB44F+hSy40g7DS1uLj1j7EgPJWV5fQH+uOEQ//X2brYfaaeqJI/PXDCFTy6ZRFlBDuv2t/Dr1Qf408ZDeHwBzp5Yzm1LJ3PdwvhmeQNRVf7p95v59er95DmyOG/KaC6cUcmHZlQyd0IpTo+P6x9eRZPTy3NfvCDiyWuAjk4/n/9FLSvrmzhv8mje39vMqMIc7vzwdD51/mSKesz4aHF5+eV7+3ji3b1d018fuGEeN583KaHH+lZdI994bhMNLSf59NLJ/MPlc7pmoqgqmw6e4Nl1B/njhkMcd3mZVlnEvVeewcfOGBPx/6TJ6eGLv1zH+3ub+dJHpzOvuoz//Zv1zBlXypO3L6a8MPe050Dw29ujb+3mhy/voNOv5GZncVZ1KedOHsW5k0dxzqRRg74SfufRdq7+yUrOmTSKJ29f0ud5AFXlvj9s4cn39jFnXAnPfvECCnPjVz23QG8ykqqysr6J/3p7D2/VNVKQk82E8nx2Nbooys3m2oXV/M2SSf3W5BMtEFDe3NnI23VNrKpvYsfRYM2/rCCHUYU5HGw9yZO3L+H8ARaeOen183c/X8O2I23ccfE0bls65ZQA31tHp59n1x3kte1H+dfr5g25zcdguDw+fvDSDn7+zl7Gl+Zz75VnsL/ZzXPrD1J/zEludhYfmzuG6xbW8JHZVQN+s/P6Atz3h808tSbYuHDhpHJ+/tnFUa0Ed9zpYe2+Ftbub2HdvhY2NJzoWv92zrgSrlkwgavnTxiwnu/0+Ljh4Xdocnp44SsXDfjvGAgoz6xr4OKZVYwri++/uQV6k/G2H2njv9/ew/5mN9ctrOaasyf0GwhHyrH2Dt7ddZyVO5tYu7+FL18yk2sXRldO8geUgOqQS1/DZe2+Fv7xdxupP+YE4Lwpo7j+nBqunDd+0Mt1qiq/Wr2ftftauH/ZmUO6uheC54u2Hmpjzd5mXth8pKvH/rmTR3HN2RO4ct54Kopy2d3kZP3+VjY0tPLBgVa2H27HF1Ce+LvFMV1FHw8W6I0xScXj8/P69kbOnFCasFkwsTjQ7GbFhkP8ccMhth9pJ0ugKNdBe+jq3+JQ64cFE8u5eFbVgN+4hoMFemOMGaK6o+38ccMhWt2dzK8pY+GkcqZVFidde5L+An3yfXc1xpgkMmtsCV+/LLYrkEdachfzjDHGxMwCvTHGpLmoAr2IXC4iO0SkXkTuifB4nog8HXp8tYhM6fHYvaHtO0Tk43EcuzHGmCgMGOhFJBt4CLgCmAvcKiJze+12O9CiqjOAB4EHQs+dC9wCnAlcDjwcej1jjDHDJJqMfjFQr6q7VdULPAUs67XPMuCJ0O1ngEsleDnbMuApVfWo6h6gPvR6xhhjhkk0gb4aONDjfkNoW8R9VNUHnAAqonwuInKHiNSKSG1jY2P0ozfGGDOgpDgZq6qPquoiVV1UVTWyV5cZY0y6iSbQHwR6LrdTE9oWcR8RcQBlwPEon2uMMSaBBrwyNhS464BLCQbpNcDfqOqWHvt8CZinql8QkVuA61X1EyJyJvBrgnX5CcCrwExV9ffz8xqBfTEcUyXQFMPzU5Udd2ax484s0Rz3ZFWNWBIZ8MpYVfWJyF3Ai0A28LiqbhGR+4FaVV0BPAY8KSL1QDPBmTaE9lsObAV8wJf6C/Kh58RUuxGR2r4uA05ndtyZxY47s8R63FG1QFDV54Hne227r8ftDuCmPp77XeC7Qx2gMcaY2CTFyVhjjDGJk46B/tGRHsAIsePOLHbcmSWm4066NsXGGGPiKx0zemOMMT1YoDfGmDSXNoF+oA6b6UJEHheRYyKyuce20SLysojsDP09aiTHmAgiMlFEXheRrSKyRUS+Etqe1scuIvki8r6IbAgd9/8NbZ8a6hRbH+ocmzvSY00EEckWkfUi8qfQ/Uw57r0isklEPhCR2tC2Ib/X0yLQR9lhM138nGAn0J7uAV5V1ZkEL0pLxw86H/B1VZ0LnA98KfR/nO7H7gEuUdWzgQXA5SJyPsEOsQ+GOsa2EOwgm46+AmzrcT9Tjhvgo6q6oMf8+SG/19Mi0BNdh820oKpvEbworaee3UOfAK4dzjENB1U9rKrrQrfbCf7yV5Pmx65BztDdnNAfBS4h2CkW0vC4AUSkBrgK+O/QfSEDjrsfQ36vp0ugj6pLZhobq6qHQ7ePAGNHcjCJFlrYZiGwmgw49lD54gPgGPAysAtoDXWKhfR9v/8I+AcgELpfQWYcNwQ/zF8SkbUickdo25Df67Y4eJpRVRWRtJ0zKyLFwO+Ar6pqWzDJC0rXYw+1DVkgIuXAc8CckR1R4onIXwPHVHWtiHxkhIczEj6kqgdFZAzwsohs7/ngYN/r6ZLRZ3qXzKMiMh4g9PexER5PQohIDsEg/ytVfTa0OSOOHUBVW4HXgaVAeajhIKTn+/1C4BoR2UuwFHsJ8GPS/7gBUNWDob+PEfxwX0wM7/V0CfRrgJmhM/K5BJuqrRjhMQ2nFcBtodu3AX8YwbEkRKg++xiwTVV/2OOhtD52EakKZfKISAHwVwTPT7wO3BjaLe2OW1XvVdUaVZ1C8Pf5NVX9JGl+3AAiUiQiJeHbwGXAZmJ4r6fNlbEiciXBml64w2ZaNlITkd8AHyHYtvQo8C3g98ByYBLBFs+fUNXeJ2xTmoh8CHgb2ER3zfYbBOv0aXvsIjKf4Im3bIKJ2XJVvV9EphHMdEcD64G/VVXPyI00cUKlm7tV9a8z4bhDx/hc6K4D+LWqfldEKhjiez1tAr0xxpjI0qV0Y4wxpg8W6I0xJs1ZoDfGmDRngd4YY9KcBXpjjElzFuiNMSbNWaA3xpg09/8BARE/YX+BP2QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Training and Validation Losses',size = 12)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e1599c5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f925adf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bccbccbc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9bf70af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n",
    "# cvscores = []\n",
    "\n",
    "# for train, val in kfold.split(x_s, classes_[y_train.argmax(1)]): \n",
    "#     model2.fit(x_s[train], y_train[train], epochs = 5, batch_size = 16, verbose=1)\n",
    "# #     scores = model2.evaluate(x_s[val], y_train[val], verbose=0)\n",
    "# #     print(\"%s: %.2f%%\" % (model2.metrics_names[1], scores[1]*100))\n",
    "# #     cvscores.append(scores[1] * 100)\n",
    "# # print(\"%.2f%% (+/- %.2f%%)\" % (np.mean(cvscores), np.std(cvscores)))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccd4d929",
   "metadata": {},
   "outputs": [],
   "source": [
    "#embeddings_weights = get_embeddings('wikipedia',300,vocabulary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d3085bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Do we have to do this?\n",
    "# x_s_ = tf.convert_to_tensor(\n",
    "#     x_s_, dtype=tf.float32, dtype_hint=None, name=None\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dc757d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# i = 0\n",
    "# j = 16\n",
    "# while j<=640:\n",
    "#     inputs = x_t[i:j]\n",
    "#     lstm = LSTM(128, return_sequences=True, return_state=True)\n",
    "#     whole_seq_output, h_0, c_0 = lstm(inputs,initial_state = [h_0, c_0])\n",
    "#     i += 16\n",
    "#     j += 16\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "478352ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# inputs = Input(batch_shape=(16,1,18))\n",
    "# lstm = LSTM(128, return_sequences=True, return_state=True)\n",
    "# whole_seq_output, h_0, c_0 = lstm(inputs)\n",
    "# model = Model(inputs=inputs, outputs=whole_seq_output)\n",
    "# k = model.predict(x_t[:640])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1da6e126",
   "metadata": {},
   "outputs": [],
   "source": [
    "# final_memory_state = tf.reshape(\n",
    "#     final_memory_state, (16,1,128), name=None\n",
    "# )\n",
    "\n",
    "# final_carry_state = tf.reshape(\n",
    "#     final_carry_state, (16,1,128), name=None\n",
    "# )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e375ad97",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# inputs2 = Input(shape=(1,sentence_maxlen,), name = 'Input')\n",
    "# #embedded_inputs = Embedding(embeddings_weights.shape[0], embeddings_weights.shape[1], weights=[embeddings_weights], name = 'Embedding')(inputs2)\n",
    "# #embedded_inputs2 = Dropout(0.2)(embedded_inputs)\n",
    "# lstm, s1,s2 = LSTM(128,return_sequences=True,return_state=True,dropout=0.3,name = 'lstm')(inputs2, initial_state = i_states)\n",
    "# #lstm = LSTM(128,return_sequences=True,dropout=0.3,name = 'lstm')(inputs2)\n",
    "# flat = Flatten(name = 'Flatten')(lstm)\n",
    "# output = (Dense(3,activation='softmax',name = 'Dense'))(flat)\n",
    "# model2 = Model(inputs=inputs2, outputs=output)\n",
    "# model2.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])    \n",
    "# model2.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59727728",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_t =np.array( [x.reshape(1,18) for x in x_t[:640]])\n",
    "\n",
    "# x_t = tf.convert_to_tensor(\n",
    "#     x_t, dtype=tf.float32, dtype_hint=None, name=None\n",
    "# )\n",
    "\n",
    "# inputs_list = [tf.squeeze(x) for x in\n",
    "#                 tf.split(1, sentence_maxlen, embedded_inputs)]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlpKernel",
   "language": "python",
   "name": "nlpkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}