deep-TR-based.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "92cf9509",
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "import re\n",
    "import os\n",
    "import nltk\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "SEED = 1013\n",
    "np.random.seed(SEED)\n",
    "#nltk.download('stopwords')\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.corpus import stopwords, twitter_samples \n",
    "from stance_utils import *\n",
    "#from parameters import *\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Dropout,Concatenate,Dense, Embedding, LSTM, SpatialDropout1D, Flatten, GRU, Bidirectional, Conv1D, Input,MaxPooling1D\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras import Model\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "stemmer = PorterStemmer()\n",
    "tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)\n",
    "stopwords_english = stopwords.words('english')\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9250cfb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = {'FAVOR': np.array([1, 0, 0]), 'AGAINST': np.array([0, 1, 0]), 'NONE': np.array([0, 0, 1])}\n",
    "classes_ = np.array(['FAVOR', 'AGAINST', 'NONE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e3dad654",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_file_m = '/data/parush/stance_mohammed/train.txt'\n",
    "test_data_file_m = '/data/parush/stance_mohammed/test.txt'\n",
    "TARGETS_m = [ 'Atheism','Climate Change is a Real Concern', 'Feminist Movement','Hillary Clinton', 'Legalization of Abortion' ]\n",
    "\n",
    "\n",
    "train_data_file_s = '/data/parush/SomasundaranWiebe-politicalDebates/train.txt'\n",
    "test_data_file_s = '/data/parush/SomasundaranWiebe-politicalDebates/test.txt'\n",
    "TARGETS_s = ['god','healthcare','guns','gayRights','abortion', 'creation']\n",
    "\n",
    "\n",
    "train_data_file_q = '/data/parush/Data_MPCHI/train.txt'\n",
    "test_data_file_q = '/data/parush/Data_MPCHI/test.txt'\n",
    "TARGETS_q = ['Are E-Cigarettes safe?','Does MMR Vaccine lead to autism in children?',\n",
    "      'Does Sunlight exposure lead to skin cancer?','Does Vitamin C prevent common cold?',\n",
    "      'Should women take HRT post-menopause?']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd466a20",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "19f54f62",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_test(train_file, test_file, target):\n",
    "    \n",
    "    sentence_maxlen = 0\n",
    "    x_train = []\n",
    "    y_train = []\n",
    "\n",
    "    \n",
    "    with open(train_file, 'r') as trainfile:\n",
    "        for line in trainfile:\n",
    "            \n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "            \n",
    "            #if line[0].strip() != 'ID' and line[1].strip() == t:\n",
    "            if line[0].strip() != 'ID' and target in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_train.append(tweet)\n",
    "                y_train.append(classes[line[3].strip()])\n",
    "    \n",
    "    with open('rich_data_feminism_athesim_abortion.txt','r') as newfile:\n",
    "        for line in newfile:\n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "            tweet = line[0]\n",
    "            tweet = process_tweet(tweet)\n",
    "            if len(tweet) > sentence_maxlen:\n",
    "                sentence_maxlen = len(tweet)\n",
    "            x_train.append(tweet)\n",
    "            y_train.append(classes[line[1].strip()])\n",
    "    \n",
    "                               \n",
    "    \n",
    "    x_test = []\n",
    "    y_test = []\n",
    "    with open(test_file, 'r') as testfile:\n",
    "        for line in testfile:\n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "        \n",
    "\n",
    "            #if line[0] != 'ID' and line[1] == t:\n",
    "            if line[0] != 'ID' and target in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_test.append(tweet)\n",
    "                y_test.append(classes[line[3].strip()])\n",
    "\n",
    "\n",
    "    \n",
    "    return x_train, y_train, x_test, y_test, sentence_maxlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2f854b53",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = train_data_file_m\n",
    "test_file = test_data_file_m\n",
    "target = TARGETS_m[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a1718156",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, y_train, x_test, y_test, sentence_maxlen = train_and_test(train_file,test_file,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4885da02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1046"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "40c96623",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words in vocab are 3140\n"
     ]
    }
   ],
   "source": [
    "vocabulary = build_vocab(x_train)\n",
    "vocab_size = len(vocabulary)\n",
    "print(\"Total words in vocab are\",vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5282b25b",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "embeddings_weights = get_embeddings('wikipedia',300,vocabulary)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3180f9ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(x_train)):\n",
    "    tweet_tensor = tweet_to_tensor(x_train[i], vocabulary)\n",
    "    if len(tweet_tensor) < sentence_maxlen:\n",
    "        diff = sentence_maxlen - len(tweet_tensor)\n",
    "        n_pad = [0]*diff\n",
    "        tweet_tensor = tweet_tensor + n_pad\n",
    "    x_train[i] = tweet_tensor\n",
    "for i in range(len(x_test)):\n",
    "    tweet_tensor = tweet_to_tensor(x_test[i], vocabulary)\n",
    "    if len(tweet_tensor) < sentence_maxlen:\n",
    "        diff = sentence_maxlen - len(tweet_tensor)\n",
    "        n_pad = [0]*diff\n",
    "        tweet_tensor = tweet_tensor + n_pad\n",
    "    x_test[i] = tweet_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7e7fe4de",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = np.array(x_train)\n",
    "\n",
    "x_test = np.array(x_test)\n",
    "\n",
    "y_train = np.asarray(y_train).astype('float32')\n",
    "y_test = np.asarray(y_test).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "148a8654",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   3,    4,    5, ...,    0,    0,    0],\n",
       "       [  17,   18,   19, ...,    0,    0,    0],\n",
       "       [  13,   28,   29, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [1895, 3136, 1892, ...,    0,    0,    0],\n",
       "       [ 421, 2120,   55, ...,    0,    0,    0],\n",
       "       [ 344,  166,  423, ...,    0,    0,    0]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "180da236",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = 3\n",
    "def biLSTM():\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(embeddings_weights.shape[0], embeddings_weights.shape[1], weights=[embeddings_weights]))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(LSTM(64,return_sequences=True,dropout=0.3))\n",
    "    model.add(Bidirectional(LSTM(64,dropout=0.3)))\n",
    "    #model.add(Flatten())\n",
    "    #add a dropout here\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes,activation='softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "def biLSTMCNN():\n",
    "    inputs = Input(shape=(sentence_maxlen,))\n",
    "    embedded_inputs = Embedding(embeddings_weights.shape[0], embeddings_weights.shape[1], weights=[embeddings_weights])(inputs)\n",
    "    embedded_inputs = Dropout(0.2)(embedded_inputs)\n",
    "    lstm = Bidirectional(LSTM(64,return_sequences=True,dropout=0.3))(embedded_inputs)\n",
    "    convs = []\n",
    "    for each_filter_size in [3,4,5]:\n",
    "        #print(rnn.shape)\n",
    "        each_conv = Conv1D(100, each_filter_size, activation='relu')(lstm)\n",
    "        each_conv = MaxPooling1D(sentence_maxlen-each_filter_size+1)(each_conv)\n",
    "        each_conv = Flatten()(each_conv)\n",
    "        #print(each_conv.shape)\n",
    "        convs.append(each_conv)\n",
    "        \n",
    "    output = Concatenate()(convs)\n",
    "    output = Dropout(0.5)(output)\n",
    "    output = (Dense(3,activation='softmax'))(output)\n",
    "    model = Model(inputs=inputs, outputs=output)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy']) \n",
    "    return model\n",
    "\n",
    "def biGRU():\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(embeddings_weights.shape[0], embeddings_weights.shape[1], weights=[embeddings_weights]))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Bidirectional(GRU(64,return_sequences=True,dropout=0.3)))\n",
    "    model.add(Bidirectional(GRU(64,dropout=0.3)))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(3,activation='softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "def biGRUCNN():\n",
    "    inputs = Input(shape=(sentence_maxlen,))\n",
    "    embedded_inputs = Embedding(embeddings_weights.shape[0], embeddings_weights.shape[1], weights=[embeddings_weights])(inputs)\n",
    "    embedded_inputs = Dropout(0.2)(embedded_inputs)\n",
    "    rnn = Bidirectional(GRU(64,return_sequences=True,dropout=0.3))(embedded_inputs)\n",
    "    convs = []\n",
    "    for each_filter_size in [3,4,5]:\n",
    "        #print(rnn.shape)\n",
    "        each_conv = Conv1D(100, each_filter_size, activation='relu')(rnn)\n",
    "        each_conv = MaxPooling1D(sentence_maxlen-each_filter_size+1)(each_conv)\n",
    "        each_conv = Flatten()(each_conv)\n",
    "        #print(each_conv.shape)\n",
    "        convs.append(each_conv)\n",
    "        \n",
    "    output = Concatenate()(convs)\n",
    "    output = Dropout(0.5)(output)\n",
    "    output = (Dense(3,activation='softmax'))(output)\n",
    "    model = Model(inputs=inputs, outputs=output)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])    \n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9437abbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-09-14 15:56:28.566754: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2021-09-14 15:56:30.789043: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 10421 MB memory:  -> device: 0, name: NVIDIA GeForce GTX 1080 Ti, pci bus id: 0000:02:00.0, compute capability: 6.1\n",
      "2021-09-14 15:56:30.790978: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:1 with 10421 MB memory:  -> device: 1, name: NVIDIA GeForce GTX 1080 Ti, pci bus id: 0000:03:00.0, compute capability: 6.1\n",
      "2021-09-14 15:56:30.792516: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:2 with 10421 MB memory:  -> device: 2, name: NVIDIA GeForce GTX 1080 Ti, pci bus id: 0000:81:00.0, compute capability: 6.1\n",
      "2021-09-14 15:56:30.794185: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:3 with 10421 MB memory:  -> device: 3, name: NVIDIA GeForce GTX 1080 Ti, pci bus id: 0000:82:00.0, compute capability: 6.1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 18)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 18, 300)      942000      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 18, 300)      0           embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   (None, 18, 128)      186880      dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d (Conv1D)                 (None, 16, 100)      38500       bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 15, 100)      51300       bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 14, 100)      64100       bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d (MaxPooling1D)    (None, 1, 100)       0           conv1d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1D)  (None, 1, 100)       0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)  (None, 1, 100)       0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 100)          0           max_pooling1d[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 100)          0           max_pooling1d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 100)          0           max_pooling1d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 300)          0           flatten[0][0]                    \n",
      "                                                                 flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 300)          0           concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 3)            903         dropout_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 1,283,683\n",
      "Trainable params: 1,283,683\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = biLSTMCNN()\n",
    "print(model.summary())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "52c8ef33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-09-14 15:56:31.618822: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n",
      "2021-09-14 15:56:35.045159: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8101\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "53/53 [==============================] - 6s 8ms/step - loss: 1.0131 - accuracy: 0.4833\n",
      "Epoch 2/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.8364 - accuracy: 0.6005\n",
      "Epoch 3/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.6580 - accuracy: 0.7249\n",
      "Epoch 4/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.4432 - accuracy: 0.8266\n",
      "Epoch 5/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.2529 - accuracy: 0.9115\n",
      "Epoch 6/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.1352 - accuracy: 0.9557\n",
      "Epoch 7/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.1035 - accuracy: 0.9605\n",
      "Epoch 8/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.1072 - accuracy: 0.9665\n",
      "Epoch 9/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0873 - accuracy: 0.9677\n",
      "Epoch 10/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0534 - accuracy: 0.9809\n",
      "Epoch 11/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0320 - accuracy: 0.9892\n",
      "Epoch 12/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0520 - accuracy: 0.9856\n",
      "Epoch 13/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0331 - accuracy: 0.9868\n",
      "Epoch 14/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0207 - accuracy: 0.9928\n",
      "Epoch 15/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0296 - accuracy: 0.9880\n",
      "Epoch 16/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0194 - accuracy: 0.9952\n",
      "Epoch 17/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0137 - accuracy: 0.9964\n",
      "Epoch 18/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0077 - accuracy: 0.9988\n",
      "Epoch 19/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0032 - accuracy: 0.9988\n",
      "Epoch 20/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0117 - accuracy: 0.9940\n",
      "Epoch 21/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0112 - accuracy: 0.9940\n",
      "Epoch 22/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0189 - accuracy: 0.9952\n",
      "Epoch 23/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0077 - accuracy: 0.9976\n",
      "Epoch 24/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0151 - accuracy: 0.9976\n",
      "Epoch 25/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0136 - accuracy: 0.9952\n",
      "Epoch 26/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0100 - accuracy: 0.9952\n",
      "Epoch 27/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0166 - accuracy: 0.9904\n",
      "Epoch 28/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0075 - accuracy: 0.9976\n",
      "Epoch 29/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0255 - accuracy: 0.9940\n",
      "Epoch 30/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0119 - accuracy: 0.9964\n",
      "Epoch 31/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0077 - accuracy: 0.9988\n",
      "Epoch 32/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0068 - accuracy: 0.9976\n",
      "Epoch 33/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0020 - accuracy: 1.0000\n",
      "Epoch 34/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0014 - accuracy: 1.0000\n",
      "Epoch 35/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0116 - accuracy: 0.9940\n",
      "Epoch 36/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0014 - accuracy: 1.0000\n",
      "Epoch 37/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0071 - accuracy: 0.9976\n",
      "Epoch 38/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0085 - accuracy: 0.9976\n",
      "Epoch 39/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0040 - accuracy: 0.9988\n",
      "Epoch 40/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0120 - accuracy: 0.9964\n",
      "Epoch 41/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0016 - accuracy: 1.0000\n",
      "Epoch 42/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.8104e-04 - accuracy: 1.0000\n",
      "Epoch 43/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.9176e-04 - accuracy: 1.0000\n",
      "Epoch 44/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.5725e-04 - accuracy: 1.0000\n",
      "Epoch 45/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0012 - accuracy: 1.0000\n",
      "Epoch 46/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0010 - accuracy: 1.0000\n",
      "Epoch 47/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0021 - accuracy: 0.9988\n",
      "Epoch 48/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0063 - accuracy: 0.9988\n",
      "Epoch 49/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0037 - accuracy: 0.9988\n",
      "Epoch 50/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0140 - accuracy: 0.9952\n",
      "accuracy: 58.57%\n",
      "Epoch 1/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.5768 - accuracy: 0.8542\n",
      "Epoch 2/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0914 - accuracy: 0.9749\n",
      "Epoch 3/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0387 - accuracy: 0.9881\n",
      "Epoch 4/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0183 - accuracy: 0.9916\n",
      "Epoch 5/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0136 - accuracy: 0.9964\n",
      "Epoch 6/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0075 - accuracy: 0.9964\n",
      "Epoch 7/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0125 - accuracy: 0.9964\n",
      "Epoch 8/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0044 - accuracy: 0.9976\n",
      "Epoch 9/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0034 - accuracy: 1.0000\n",
      "Epoch 10/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0041 - accuracy: 0.9988\n",
      "Epoch 11/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0019 - accuracy: 1.0000\n",
      "Epoch 12/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0015 - accuracy: 1.0000\n",
      "Epoch 13/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0013 - accuracy: 1.0000\n",
      "Epoch 14/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0011 - accuracy: 1.0000\n",
      "Epoch 15/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0010 - accuracy: 1.0000\n",
      "Epoch 16/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0024 - accuracy: 0.9988\n",
      "Epoch 17/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0028 - accuracy: 0.9988\n",
      "Epoch 18/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0041 - accuracy: 0.9988\n",
      "Epoch 19/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0048 - accuracy: 0.9976\n",
      "Epoch 20/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0014 - accuracy: 1.0000\n",
      "Epoch 21/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0012 - accuracy: 1.0000\n",
      "Epoch 22/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.8202e-04 - accuracy: 1.0000\n",
      "Epoch 23/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 5.5750e-04 - accuracy: 1.0000\n",
      "Epoch 24/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 7.6785e-04 - accuracy: 1.0000\n",
      "Epoch 25/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 7.2774e-04 - accuracy: 1.0000\n",
      "Epoch 26/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.7382e-04 - accuracy: 1.0000\n",
      "Epoch 27/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0059 - accuracy: 0.9988\n",
      "Epoch 28/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0058 - accuracy: 0.9988\n",
      "Epoch 29/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0103 - accuracy: 0.9988\n",
      "Epoch 30/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0015 - accuracy: 1.0000\n",
      "Epoch 31/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.4015e-04 - accuracy: 1.0000\n",
      "Epoch 32/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.8271e-04 - accuracy: 1.0000\n",
      "Epoch 33/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0018 - accuracy: 0.9988\n",
      "Epoch 34/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.1736e-04 - accuracy: 1.0000\n",
      "Epoch 35/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.9919e-04 - accuracy: 1.0000\n",
      "Epoch 36/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0013 - accuracy: 0.9988\n",
      "Epoch 37/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0016 - accuracy: 0.9988\n",
      "Epoch 38/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.3501e-04 - accuracy: 1.0000\n",
      "Epoch 39/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.5608e-04 - accuracy: 1.0000\n",
      "Epoch 40/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.2221e-04 - accuracy: 1.0000\n",
      "Epoch 41/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.1393e-04 - accuracy: 1.0000\n",
      "Epoch 42/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.6378e-05 - accuracy: 1.0000\n",
      "Epoch 43/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 8.9080e-05 - accuracy: 1.0000\n",
      "Epoch 44/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0019 - accuracy: 0.9976\n",
      "Epoch 45/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0019 - accuracy: 0.9988\n",
      "Epoch 46/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0116 - accuracy: 0.9988\n",
      "Epoch 47/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0015 - accuracy: 1.0000\n",
      "Epoch 48/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.3876e-04 - accuracy: 1.0000\n",
      "Epoch 49/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.4552e-04 - accuracy: 1.0000\n",
      "Epoch 50/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.5060e-04 - accuracy: 1.0000\n",
      "accuracy: 98.56%\n",
      "Epoch 1/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0433 - accuracy: 0.9904\n",
      "Epoch 2/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0458 - accuracy: 0.9869\n",
      "Epoch 3/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0253 - accuracy: 0.9928\n",
      "Epoch 4/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0101 - accuracy: 0.9952\n",
      "Epoch 5/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0024 - accuracy: 0.9988\n",
      "Epoch 6/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0080 - accuracy: 0.9988\n",
      "Epoch 7/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.8629e-04 - accuracy: 1.0000\n",
      "Epoch 8/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.7693e-04 - accuracy: 1.0000\n",
      "Epoch 9/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0010 - accuracy: 1.0000\n",
      "Epoch 10/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.9649e-04 - accuracy: 1.0000\n",
      "Epoch 11/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0086 - accuracy: 0.9976\n",
      "Epoch 12/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0013 - accuracy: 1.0000\n",
      "Epoch 13/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.0857e-04 - accuracy: 1.0000\n",
      "Epoch 14/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.2258e-04 - accuracy: 1.0000\n",
      "Epoch 15/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0062 - accuracy: 0.9976\n",
      "Epoch 16/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0016 - accuracy: 1.0000\n",
      "Epoch 17/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0022 - accuracy: 0.9988\n",
      "Epoch 18/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.4773e-04 - accuracy: 1.0000\n",
      "Epoch 19/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.4201e-04 - accuracy: 1.0000\n",
      "Epoch 20/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.3162e-04 - accuracy: 1.0000\n",
      "Epoch 21/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.5172e-04 - accuracy: 1.0000\n",
      "Epoch 22/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 2.5708e-05 - accuracy: 1.0000\n",
      "Epoch 23/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 3.9995e-05 - accuracy: 1.0000\n",
      "Epoch 24/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 4.1962e-04 - accuracy: 1.0000\n",
      "Epoch 25/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 5.5962e-05 - accuracy: 1.0000\n",
      "Epoch 26/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.3834e-05 - accuracy: 1.0000\n",
      "Epoch 27/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.6001e-04 - accuracy: 1.0000\n",
      "Epoch 28/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 7.5916e-05 - accuracy: 1.0000\n",
      "Epoch 29/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.5446e-05 - accuracy: 1.0000\n",
      "Epoch 30/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.6972e-05 - accuracy: 1.0000\n",
      "Epoch 31/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.1364e-05 - accuracy: 1.0000\n",
      "Epoch 32/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.2138e-04 - accuracy: 1.0000\n",
      "Epoch 33/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 8.2567e-04 - accuracy: 1.0000\n",
      "Epoch 34/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 7.3636e-05 - accuracy: 1.0000\n",
      "Epoch 35/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.1813e-04 - accuracy: 1.0000\n",
      "Epoch 36/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0087 - accuracy: 0.9976\n",
      "Epoch 37/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0077 - accuracy: 0.9976\n",
      "Epoch 38/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.1003e-04 - accuracy: 1.0000\n",
      "Epoch 39/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.0206e-05 - accuracy: 1.0000\n",
      "Epoch 40/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0196 - accuracy: 0.9952\n",
      "Epoch 41/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0184 - accuracy: 0.9940\n",
      "Epoch 42/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0282 - accuracy: 0.9904\n",
      "Epoch 43/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0184 - accuracy: 0.9952\n",
      "Epoch 44/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0108 - accuracy: 0.9988\n",
      "Epoch 45/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.3793e-04 - accuracy: 1.0000\n",
      "Epoch 46/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0127 - accuracy: 0.9964\n",
      "Epoch 47/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0064 - accuracy: 0.9988\n",
      "Epoch 48/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0051 - accuracy: 0.9988\n",
      "Epoch 49/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0011 - accuracy: 1.0000\n",
      "Epoch 50/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.7165e-04 - accuracy: 1.0000\n",
      "accuracy: 100.00%\n",
      "Epoch 1/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0013 - accuracy: 0.9988\n",
      "Epoch 2/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0035 - accuracy: 0.9988\n",
      "Epoch 3/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0179 - accuracy: 0.9976\n",
      "Epoch 4/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0047 - accuracy: 0.9988\n",
      "Epoch 5/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0075 - accuracy: 0.9976\n",
      "Epoch 6/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 3.4307e-04 - accuracy: 1.0000\n",
      "Epoch 7/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.6771e-05 - accuracy: 1.0000\n",
      "Epoch 8/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.9633e-05 - accuracy: 1.0000\n",
      "Epoch 9/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0083 - accuracy: 0.9976\n",
      "Epoch 10/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0061 - accuracy: 0.9976\n",
      "Epoch 11/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.5004e-04 - accuracy: 1.0000\n",
      "Epoch 12/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0061 - accuracy: 0.9988\n",
      "Epoch 13/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0156 - accuracy: 0.9964\n",
      "Epoch 14/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 9.5927e-04 - accuracy: 1.0000\n",
      "Epoch 15/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.3897e-04 - accuracy: 1.0000\n",
      "Epoch 16/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.4226e-04 - accuracy: 1.0000\n",
      "Epoch 17/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.5091e-05 - accuracy: 1.0000\n",
      "Epoch 18/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.3504e-05 - accuracy: 1.0000\n",
      "Epoch 19/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.7618e-05 - accuracy: 1.0000\n",
      "Epoch 20/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 5.7452e-05 - accuracy: 1.0000\n",
      "Epoch 21/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 2.3224e-04 - accuracy: 1.0000\n",
      "Epoch 22/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0050 - accuracy: 0.9988\n",
      "Epoch 23/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 2.4386e-04 - accuracy: 1.0000\n",
      "Epoch 24/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 2.6531e-05 - accuracy: 1.0000\n",
      "Epoch 25/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0026 - accuracy: 0.9988\n",
      "Epoch 26/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.6151e-05 - accuracy: 1.0000\n",
      "Epoch 27/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.0968e-04 - accuracy: 1.0000\n",
      "Epoch 28/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 5.5373e-05 - accuracy: 1.0000\n",
      "Epoch 29/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 2.0577e-05 - accuracy: 1.0000\n",
      "Epoch 30/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 1.5826e-04 - accuracy: 1.0000\n",
      "Epoch 31/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.3341e-06 - accuracy: 1.0000\n",
      "Epoch 32/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 9.0779e-06 - accuracy: 1.0000\n",
      "Epoch 33/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 8.6960e-06 - accuracy: 1.0000\n",
      "Epoch 34/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.4842e-06 - accuracy: 1.0000\n",
      "Epoch 35/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 6.3002e-05 - accuracy: 1.0000\n",
      "Epoch 36/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.1130e-04 - accuracy: 1.0000\n",
      "Epoch 37/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.0955e-05 - accuracy: 1.0000\n",
      "Epoch 38/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.0076e-05 - accuracy: 1.0000\n",
      "Epoch 39/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.6269e-05 - accuracy: 1.0000\n",
      "Epoch 40/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.4648e-06 - accuracy: 1.0000\n",
      "Epoch 41/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 7.2881e-06 - accuracy: 1.0000\n",
      "Epoch 42/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.0207e-05 - accuracy: 1.0000\n",
      "Epoch 43/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.6934e-06 - accuracy: 1.0000\n",
      "Epoch 44/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.0089e-05 - accuracy: 1.0000\n",
      "Epoch 45/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 9.3646e-05 - accuracy: 1.0000\n",
      "Epoch 46/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.3183e-05 - accuracy: 1.0000\n",
      "Epoch 47/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.0063e-06 - accuracy: 1.0000\n",
      "Epoch 48/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 5.9277e-05 - accuracy: 1.0000\n",
      "Epoch 49/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.5497e-05 - accuracy: 1.0000\n",
      "Epoch 50/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 5.7079e-04 - accuracy: 1.0000\n",
      "accuracy: 100.00%\n",
      "Epoch 1/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0030 - accuracy: 0.9988\n",
      "Epoch 2/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0026 - accuracy: 0.9988\n",
      "Epoch 3/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 9.5349e-04 - accuracy: 1.0000\n",
      "Epoch 4/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 0.0027 - accuracy: 0.9988\n",
      "Epoch 5/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.1988e-05 - accuracy: 1.0000\n",
      "Epoch 6/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 5.7471e-05 - accuracy: 1.0000\n",
      "Epoch 7/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 7.1939e-06 - accuracy: 1.0000\n",
      "Epoch 8/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 9.4467e-05 - accuracy: 1.0000\n",
      "Epoch 9/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 1.4431e-05 - accuracy: 1.0000\n",
      "Epoch 10/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 1.1859e-04 - accuracy: 1.0000\n",
      "Epoch 11/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 5.7931e-04 - accuracy: 1.0000\n",
      "Epoch 12/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0064 - accuracy: 0.9976\n",
      "Epoch 13/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0036 - accuracy: 0.9988\n",
      "Epoch 14/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.4025e-04 - accuracy: 1.0000\n",
      "Epoch 15/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.9674e-05 - accuracy: 1.0000\n",
      "Epoch 16/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 7.8678e-05 - accuracy: 1.0000\n",
      "Epoch 17/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 8.3680e-05 - accuracy: 1.0000\n",
      "Epoch 18/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 5.2016e-04 - accuracy: 1.0000\n",
      "Epoch 19/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 6.4226e-04 - accuracy: 1.0000\n",
      "Epoch 20/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.3107e-06 - accuracy: 1.0000\n",
      "Epoch 21/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 5.2040e-05 - accuracy: 1.0000\n",
      "Epoch 22/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.6064e-06 - accuracy: 1.0000\n",
      "Epoch 23/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.0749e-06 - accuracy: 1.0000\n",
      "Epoch 24/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.0426e-04 - accuracy: 1.0000\n",
      "Epoch 25/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.6034e-07 - accuracy: 1.0000\n",
      "Epoch 26/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.6599e-04 - accuracy: 1.0000\n",
      "Epoch 27/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 1.6158e-06 - accuracy: 1.0000\n",
      "Epoch 28/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 3.2343e-07 - accuracy: 1.0000\n",
      "Epoch 29/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 1.0274e-05 - accuracy: 1.0000\n",
      "Epoch 30/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 9.4131e-07 - accuracy: 1.0000\n",
      "Epoch 31/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 2.6460e-06 - accuracy: 1.0000\n",
      "Epoch 32/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 5.5643e-07 - accuracy: 1.0000\n",
      "Epoch 33/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 3.7789e-05 - accuracy: 1.0000\n",
      "Epoch 34/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 1.0481e-06 - accuracy: 1.0000\n",
      "Epoch 35/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 1.1115e-06 - accuracy: 1.0000\n",
      "Epoch 36/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 2.2331e-07 - accuracy: 1.0000\n",
      "Epoch 37/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.3483e-06 - accuracy: 1.0000\n",
      "Epoch 38/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.0184e-06 - accuracy: 1.0000\n",
      "Epoch 39/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 4.6668e-06 - accuracy: 1.0000\n",
      "Epoch 40/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.3481e-04 - accuracy: 1.0000\n",
      "Epoch 41/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 3.6687e-07 - accuracy: 1.0000\n",
      "Epoch 42/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 1.0506e-06 - accuracy: 1.0000\n",
      "Epoch 43/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.5792e-07 - accuracy: 1.0000\n",
      "Epoch 44/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 2.1783e-06 - accuracy: 1.0000\n",
      "Epoch 45/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 1.8301e-06 - accuracy: 1.0000\n",
      "Epoch 46/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0023 - accuracy: 0.9988\n",
      "Epoch 47/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0208 - accuracy: 0.9952\n",
      "Epoch 48/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0486 - accuracy: 0.9940\n",
      "Epoch 49/50\n",
      "53/53 [==============================] - 0s 7ms/step - loss: 0.0226 - accuracy: 0.9988\n",
      "Epoch 50/50\n",
      "53/53 [==============================] - 0s 6ms/step - loss: 3.1493e-05 - accuracy: 1.0000\n",
      "accuracy: 99.52%\n",
      "91.33% (+/- 16.39%)\n"
     ]
    }
   ],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\n",
    "cvscores = []\n",
    "\n",
    "for train, val in kfold.split(x_train, classes_[y_train.argmax(1)]):\n",
    "    model.fit(x_train[train], y_train[train], epochs = 50, batch_size = 16, verbose=1)\n",
    "    scores = model.evaluate(x_train[val], y_train[val], verbose=0)\n",
    "    print(\"%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))\n",
    "    cvscores.append(scores[1] * 100)\n",
    "print(\"%.2f%% (+/- %.2f%%)\" % (np.mean(cvscores), np.std(cvscores)))\n",
    "\n",
    "y_pred = np.round(model.predict(x_test))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "63508d8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained on Feminist Movement +  and tested on Feminist Movement \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0     0.3218    0.4828    0.3862        58\n",
      "           1     0.7362    0.6557    0.6936       183\n",
      "\n",
      "   micro avg     0.5920    0.6141    0.6029       241\n",
      "   macro avg     0.5290    0.5692    0.5399       241\n",
      "weighted avg     0.6365    0.6141    0.6197       241\n",
      " samples avg     0.5193    0.5193    0.5193       241\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print(\"Trained on {} +  and tested on {} \\n\".format(target,target),classification_report(y_test, y_pred, digits=4, labels = [0,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7a76a961-c89f-478e-9260-85dc32972a5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Trained on {} + Siamese prediction on HRT as Feminism -  and tested on {} \\n\".format(target,target),classification_report(y_test, y_pred, digits=4, labels = [0,1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53c4abcf-87d3-4717-b3e9-1ef66f56efc5",
   "metadata": {},
   "source": [
    "# Getting test data from another datset. We will use above model to predict here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "87b3c79b-a0ef-44b5-a3e9-d8bd3cd0de4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_train_file = train_data_file_m\n",
    "new_test_file = test_data_file_m\n",
    "new_target = TARGETS_m[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f6d11628",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "_, _, x_test_, y_test_, _ = train_and_test(new_train_file, new_test_file, new_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "519236c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(x_test_)):\n",
    "    tweet_tensor_ = tweet_to_tensor(x_test_[i], vocabulary)\n",
    "    if len(tweet_tensor_) < sentence_maxlen:\n",
    "        diff = sentence_maxlen - len(tweet_tensor_)\n",
    "        n_pad = [0]*diff\n",
    "        tweet_tensor_ = tweet_tensor_ + n_pad\n",
    "    else:\n",
    "        tweet_tensor_ = tweet_tensor_[:sentence_maxlen]\n",
    "    x_test_[i] = tweet_tensor_\n",
    "\n",
    "\n",
    "x_test_ = np.array(x_test_)\n",
    "y_test_ = np.asarray(y_test_).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5007531e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained on Feminist Movement and tested on Legalization of Abortion \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0     0.2899    0.4348    0.3478        46\n",
      "           1     0.7610    0.6402    0.6954       189\n",
      "\n",
      "   micro avg     0.6184    0.6000    0.6091       235\n",
      "   macro avg     0.5254    0.5375    0.5216       235\n",
      "weighted avg     0.6688    0.6000    0.6274       235\n",
      " samples avg     0.5036    0.5036    0.5036       235\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/p/parush/.conda/envs/nlp/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "y_pred_ = np.round(model.predict(x_test_))\n",
    "print(\"Trained on {} and tested on {} \\n\".format(target,new_target),classification_report(y_test_, y_pred_, digits=4, labels = [0,1]))\n",
    "#print(\"Trained on {} + Siamese prediction on HRT as Feminism -  and tested on {} \\n\".format(target,new_target),classification_report(y_test_, y_pred_, digits=4, labels = [0,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c52a3897",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b324bccf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlpKernel",
   "language": "python",
   "name": "nlpkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}