lstm_v4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "lstm-v4.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "455AEdyT9NkN"
      ],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNNhQ8+19/UeyggunZoiwdI",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/puaqieshang/automatic-labeling-heart-vessels/blob/master/lstm_v4.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VGtoKl-2PRm6",
        "colab_type": "text"
      },
      "source": [
        "# Labeling Coronary Arteries using LSTM \n",
        "\n",
        "Update from previous version:\n",
        "\n",
        "* Scaled data using StandardScaler() \n",
        "* Otherwise use Spherical Transform \n",
        "\n",
        "\n",
        "This version is able to run remotely on NCI"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-JURHhvqTx4W",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "Before continuing, please refer to this [thread](https://stackoverflow.com/questions/38714959/understanding-keras-lstms/50235563#50235563) followed by this [thread](https://stackoverflow.com/questions/53955093/doubts-regarding-understanding-keras-lstms) to understand LSTM better "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2aeYJgDoeUN4",
        "colab_type": "code",
        "outputId": "2accbf99-88ad-4024-81e7-694e21eac99e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Import modules\n",
        "import tensorflow\n",
        "from tensorflow.keras.layers import Dense, Dropout, LSTM, Embedding, Bidirectional\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import random\n",
        "\n",
        "import tensorflow as tf\n",
        "tf.__version__\n",
        "\n",
        "# Does keras support cuda?"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2.2.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a9rYiRbBd63-",
        "colab_type": "text"
      },
      "source": [
        "# Data Retrieval\n",
        "\n",
        "Different Classes of Vessels and its labels\n",
        "<table>\n",
        "  <tr>\n",
        "    <th>Label</th>\n",
        "    <th>Class (Vessels)</th>\n",
        "  </tr>\n",
        "  <tr>\n",
        "    <td>0</td>\n",
        "    <td>LAD</td>\n",
        "  </tr>\n",
        "  <tr>\n",
        "    <td>1</td>\n",
        "    <td>Diagonals</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>2</td>\n",
        "    <td>Septals</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>3</td>\n",
        "    <td>LCX</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>4</td>\n",
        "    <td>Obtuse Marginal</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>5</td>\n",
        "    <td>Atrials</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>6</td>\n",
        "    <td>LCIM</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>7</td>\n",
        "    <td>Acutes</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>8</td>\n",
        "    <td>Crux</td>\n",
        "  </tr>\n",
        "</table>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K_95J3-fd9jX",
        "colab_type": "text"
      },
      "source": [
        "## Import from Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0jVxpn36SD2P",
        "colab_type": "code",
        "outputId": "45fc5c19-c9ae-4d97-8c9f-accd796b3317",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "# Load the Drive helper and mount\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# Add exclaimation mark to type code in terminal\n",
        "!unzip -uq \"/content/drive/My Drive/processed-rnn.zip\"\n",
        "!ls \"/content/\" # Lists all the files "
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n",
            "drive  processed  sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SVrD4JJ-ekQ9",
        "colab_type": "text"
      },
      "source": [
        "## Store Data on Pandas\n",
        "\n",
        "There are 2 methods to store data:\n",
        "\n",
        "1.   Load data from a **single** text file/heart e.g. AHG0004.txt then feed it into the model. Use a for loop to repeat for other text files\n",
        "2.   Load data from **all** text files in insert into a single pandas dataframe \n",
        "\n",
        "After loading the data, it is recommended to scale data for MLP, check out the different scaling methods [here](https://towardsdatascience.com/normalization-vs-standardization-quantitative-analysis-a91e8a79cebf).\n",
        "\n",
        "---\n",
        "Edit:\n",
        "*   ~~Training model with all heart datasets using option 2~~\n",
        "*   ~~Testing model with a **single** dataset using option 1~~\n",
        "*   Now both train and test uses Option 1, its like the model is learning from 324 different datasets "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CZKVG1f_BdfF",
        "colab_type": "text"
      },
      "source": [
        "### Option 1 - Single Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vla_y540XFlO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.preprocessing import StandardScaler \n",
        "\n",
        "def load_data_single(input_file, scalerType):\n",
        "    \n",
        "    # pri nt ('Loading data...')\n",
        "    df = pd.read_csv(input_file, header=None, sep=' ', usecols=[1,2,3,4,5], \n",
        "                     names=['x','y', 'z', 'radius', 'labels'])\n",
        "\n",
        "    # train_size = int(len(df) * (1 - test_split))\n",
        "    \n",
        "    # df = df.sample(frac=1)\n",
        "    # df = df.sample(frac=1).reset_index(drop=True) # shuffle and reset row index\n",
        "    # print(df.head())\n",
        "\n",
        "    X = df[['x','y', 'z', 'radius']].values\n",
        "    # print(X)\n",
        "    df['labels'] = df['labels'].astype(int) #convert to int\n",
        "    y = np.array(df['labels'].values)\n",
        "\n",
        "    if scalerType is \"spherical\":\n",
        "        print(\"Gimme a sec\")\n",
        "\n",
        "    else: # Standard Scaling\n",
        "        scaler = StandardScaler() # can test other scaling methods, would not recommend normalisation\n",
        "        scaler = scaler.fit(X) \n",
        "\n",
        "    standardized_X = scaler.transform(X)\n",
        "    # print(standardized_X)\n",
        "\n",
        "    # X_test = np.array(df['sequence'].values[train_size:])\n",
        "    # print(y_test)\n",
        "    y = to_categorical(y, num_classes=9)\n",
        "    # X = pad_sequences(standardized_X) # SHOULD I REMOVE THIS?\n",
        "    # print(X)\n",
        "\n",
        "    return standardized_X, y\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "455AEdyT9NkN",
        "colab_type": "text"
      },
      "source": [
        "### Option 2 - Entire dataset\n",
        "Combining all heart datasets into a single pandas dataframe"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O1PM8mu58z-k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_data_all(folder_path):\n",
        "    \n",
        "    print ('Loading data...')\n",
        "    columns = ['x','y', 'z', 'radius', 'labels']\n",
        "    df = pd.DataFrame(columns=columns)\n",
        "\n",
        "    folder = os.listdir(folder_path)\n",
        "    # counter = 1\n",
        "\n",
        "    for singleFile in folder:\n",
        "        \n",
        "        # print(f\"{counter}. Currently retrieving data from {singleFile}\")\n",
        "        single_heart_df = pd.read_csv(dest+singleFile, header=None, \n",
        "                                      sep=' ', usecols=[1,2,3,4,5], \n",
        "                                      names=columns)\n",
        "\n",
        "        # train_size = int(len(df) * (1 - test_split))\n",
        "        # counter += 1\n",
        "        df = df.append(single_heart_df, ignore_index=True) \n",
        "        \n",
        "    # df = df.sample(frac=1).reset_index(drop=True) # shuffle and reset row index\n",
        "    print(f\"The dataset has a shape of {df.shape}\\n\")\n",
        "    \n",
        "    print(df.head())\n",
        "    X = df[['x','y', 'z', 'radius']].values\n",
        "    df['labels'] = df['labels'].astype(int) #convert to int\n",
        "    y = np.array(df['labels'].values)\n",
        "    # print(X_train.shape)\n",
        "    # print(y_train)\n",
        "    # X_test = np.array(df['sequence'].values[train_size:])\n",
        "    # y_test = np.array(df['target'].values[train_size:])\n",
        "\n",
        "    y = to_categorical(y, num_classes=9)\n",
        "    X = pad_sequences(X)\n",
        "    # X = np.reshape(X, (1, X.shape[0], X.shape[1]))\n",
        "\n",
        "    return X, y\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i_7NWB84nxDF",
        "colab_type": "text"
      },
      "source": [
        "## Create Dataset\n",
        "There are 2 ways of doing this - I'm not sure which one is more superior:\n",
        "\n",
        "1.   Split ~~entire~~ a single dataframe into overlapping sequence or sliding windows\n",
        "2.   [Divide the ~~entire~~ single heart dataframe into non-overlapping sequence](https://machinelearningmastery.com/prepare-univariate-time-series-data-long-short-term-memory-networks/)\n",
        "\n",
        "\n",
        "Edit: This [thread](https://https://datascience.stackexchange.com/questions/27628/sliding-window-leads-to-overfitting-in-lstm) explains perfectly. Going for option 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8QMP3HazAalD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_dataset(X_dataset, y_dataset, look_back): # look_back = how many time steps\n",
        "    X, y = [], []\n",
        "    \n",
        "    for i in range (len(X_dataset)-look_back):\n",
        "        \n",
        "        X.append(X_dataset[i:(i+look_back)])\n",
        "        y.append(y_dataset[i+look_back])\n",
        "    return np.array(X), np.array(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gfo3TqM5SHCg",
        "colab_type": "code",
        "outputId": "223b2166-24ef-4e67-c0a6-ddb6ab39c0ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "input_file = \"/content/processed/AP1113.txt\"\n",
        "\n",
        "X_train, y_train = load_data_single(input_file, \"standard\")\n",
        "\n",
        "print(\"Before creating dataset\")\n",
        "print(X_train.shape)\n",
        "print(y_train.shape)\n",
        "print(y_train[0:9])"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Before creating dataset\n",
            "(20429, 4)\n",
            "(20429, 9)\n",
            "[[0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 1.]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gt4f4ljEN_8k",
        "colab_type": "text"
      },
      "source": [
        "Creates a sequence of 20 points"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l8kl60nFOtff",
        "colab_type": "code",
        "outputId": "34a6245b-7f02-4fa5-c3a6-ced98d82c7c1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "SEQ_LENGTH = 20\n",
        "look_back = SEQ_LENGTH\n",
        "X_train, y_train = create_dataset(X_train, y_train, look_back)\n",
        "print(\"After creating dataset\")\n",
        "print(X_train.shape)\n",
        "print(y_train.shape)\n",
        "print(y_train[0:1])"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "After creating dataset\n",
            "(20409, 20, 4)\n",
            "(20409, 9)\n",
            "[[0. 0. 0. 0. 0. 0. 0. 0. 1.]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "axDU9abKK1Ml",
        "colab_type": "code",
        "outputId": "8c176417-acb8-4899-f202-158dd847d3b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "print(X_train[0])"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[-0.20530254 -0.61929728  0.75777617  0.56862264]\n",
            " [-0.20665676 -0.62014886  0.75730993  0.56873322]\n",
            " [-0.20800768 -0.62099581  0.75684368  0.56895437]\n",
            " [-0.20936191 -0.62184276  0.75637744  0.56906494]\n",
            " [-0.21071613 -0.62268971  0.75591119  0.56928609]\n",
            " [-0.21206706 -0.62354128  0.75544495  0.56950724]\n",
            " [-0.21342128 -0.62438823  0.7549787   0.56972839]\n",
            " [-0.2147722  -0.62523518  0.75451246  0.56994954]\n",
            " [-0.21612643 -0.62608213  0.75404621  0.57017069]\n",
            " [-0.21748065 -0.62693371  0.75357997  0.57039184]\n",
            " [-0.21883157 -0.62778066  0.75311372  0.57061299]\n",
            " [-0.21986211 -0.62802595  0.75319849  0.57083414]\n",
            " [-0.22089264 -0.62826199  0.75324088  0.57105529]\n",
            " [-0.22192317 -0.62851191  0.75328327  0.57116586]\n",
            " [-0.2229504  -0.62877571  0.75336804  0.57138701]\n",
            " [-0.22397763 -0.6290534   0.75341042  0.57160816]\n",
            " [-0.22500156 -0.6293496   0.75345281  0.57182931]\n",
            " [-0.22602218 -0.62965969  0.75353758  0.57205046]\n",
            " [-0.2270395  -0.62999291  0.75357997  0.57227161]\n",
            " [-0.22805352 -0.63034002  0.75362235  0.57238219]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QApxMGvxepqc",
        "colab_type": "text"
      },
      "source": [
        "# Model Definition\n",
        "\n",
        "If not sure what the input shape should be in the first layer, refer to these websites:\n",
        "*   [1D input for Time Series Prediction](https://machinelearningmastery.com/time-series-prediction-lstm-recurrent-neural-networks-python-keras/)\n",
        "*   [2D input](https://stackoverflow.com/questions/46207004/keras-lstm-dense-layer-multidimensional-input)\n",
        "\n",
        "Should the first layer be Dense or LSTM?\n",
        "[Answer](https://github.com/keras-team/keras/issues/2673)\n",
        "\n",
        "Edit 28/5/2020: Reduced number of Bidirectional layers\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "15t3RdkGL6fp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.compat.v1.keras.layers import CuDNNLSTM"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jkdw-wXKehIt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_model(look_back):\n",
        "    print ('Creating model...')\n",
        "    model = Sequential()\n",
        "    # as first layer in a sequential model:\n",
        "    #(batchsize, timesteps, features)\n",
        "\n",
        "    model.add(Bidirectional(CuDNNLSTM(128, return_sequences=True),\n",
        "                            input_shape=(look_back, 4))) \n",
        "    model.add(Dropout(0.2))\n",
        "    model.add(Bidirectional(CuDNNLSTM(128, return_sequences=False))) \n",
        "    model.add(Dropout(0.2))\n",
        "\n",
        "    model.add(Dense(64))\n",
        "    # model.add(Bidirectional(CuDNNLSTM(100, return_sequences=True)))\n",
        "    # model.add(Bidirectional(LSTM(output_dim=64, activation='sigmoid', inner_activation='hard_sigmoid')))\n",
        "    # model.add(Dropout(0.2))\n",
        "\n",
        "    model.add(Dense(32))\n",
        "    # model.add(Bidirectional(CuDNNLSTM(100, return_sequences=True)))\n",
        "    # model.add(Bidirectional(LSTM(output_dim=32, activation='sigmoid', inner_activation='hard_sigmoid')))\n",
        "    # model.add(Dropout(0.2))\n",
        "\n",
        "    model.add(Dense(16))\n",
        "    # model.add(Bidirectional(CuDNNLSTM(9, return_sequences=False)))\n",
        "    # model.add(Bidirectional(LSTM(output_dim=16, activation='sigmoid', inner_activation='hard_sigmoid')))\n",
        "    # model.add(Dropout(0.2))\n",
        "\n",
        "\n",
        "    model.add(Dense(9, activation='softmax'))\n",
        "    # model.add(Dense(1, activation='softmax'))\n",
        "    \n",
        "\n",
        "    print ('Compiling...')\n",
        "    model.compile(loss='categorical_crossentropy',\n",
        "                  optimizer='Adam',\n",
        "                  metrics=['accuracy'])\n",
        "    \n",
        "    # model.build(input_shape)\n",
        "\n",
        "    print(model.summary())\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GbVC7W4bVbz1",
        "colab_type": "code",
        "outputId": "20f5aef9-8ea7-4649-e11b-665db6c51eea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        }
      },
      "source": [
        "look_back = SEQ_LENGTH\n",
        "model = create_model(look_back)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Creating model...\n",
            "Compiling...\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "bidirectional (Bidirectional (None, 20, 256)           137216    \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 20, 256)           0         \n",
            "_________________________________________________________________\n",
            "bidirectional_1 (Bidirection (None, 256)               395264    \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 256)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 64)                16448     \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 32)                2080      \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 16)                528       \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 9)                 153       \n",
            "=================================================================\n",
            "Total params: 551,689\n",
            "Trainable params: 551,689\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PLN4BfbTeucU",
        "colab_type": "text"
      },
      "source": [
        "# Train and Validate Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fK9Yv8jUkPRV",
        "colab_type": "text"
      },
      "source": [
        "Should I shuffle my data?\n",
        "\n",
        "[Solution 1](https://stackoverflow.com/questions/44788946/shuffling-training-data-with-lstm-rnn):\n",
        "You should shuffle the training data (a set of fixed-length sequences) when you shuffle the order in which sequences are fed to the RNN. YOU SHOULD NOT SHUFFLE THE ORDERING OF POINTS WITHIN A SEQUENCE - otherwise whats the point of using LSTM, you might as well use a MLP. \n",
        "\n",
        "\n",
        "Edit: Solved NCI issue. Now just wondering why does the val loss varies so much\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YuxpymwCDsQX",
        "colab_type": "code",
        "outputId": "014035aa-d865-47d3-c0f8-0a9bbd65bc08",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Case where data is shuffled\n",
        "look_back = SEQ_LENGTH\n",
        "NO_OF_SAMPLES = 300\n",
        "\n",
        "def train_and_validate(input_folder_path):\n",
        "    \n",
        "    print(\"Training model...\")\n",
        "    train_loss = []\n",
        "    train_acc = []\n",
        "    val_loss = []\n",
        "    val_acc = []\n",
        "\n",
        "    folder = os.listdir(input_folder_path)\n",
        "    # print(len(folder))\n",
        "    # counter = 1\n",
        "\n",
        "    for counter in range(0, NO_OF_SAMPLES):\n",
        "        i = random.randint(0, len(folder)-1) # random might be bad?\n",
        "        input_file_path = input_folder_path+folder[i]\n",
        "        print(f\"{counter+1}. Now training on {folder[i]}\")\n",
        "    # for singleFile in folder:\n",
        "    #     print(f\"Heart No.{counter} - {singleFile}\")\n",
        "    #     input_file_path = input_folder_path + singleFile   \n",
        "        X_train, y_train = load_data_single(input_file_path, \"standard\")\n",
        "        # print(f\"{counter}. Currently retrieving data from {singleFile}\")\n",
        "        X_train, y_train = create_dataset(X_train, y_train, look_back)\n",
        "\n",
        "\n",
        "       # 0.1 rows are used for validation\n",
        "        hist = model.fit(X_train, y_train, \n",
        "                        batch_size=64, epochs=1, \n",
        "                        shuffle = True, # shuffle the sequences, but not the individual data\n",
        "                        validation_split = 0.1, verbose = 2)\n",
        "\n",
        "        train_loss.append(hist.history['loss'][0])\n",
        "        train_acc.append(hist.history['accuracy'][0])\n",
        "        val_loss.append(hist.history['val_loss'][0])\n",
        "        val_acc.append(hist.history['val_accuracy'][0])\n",
        "        counter += 1\n",
        "\n",
        "    print(\"Done Training.\")\n",
        "    return train_loss, train_acc, val_loss, val_acc\n",
        "\n",
        "\n",
        "folder_path = \"/content/processed/\" # The only path that I have to change for NCI\n",
        "train_loss, train_acc, val_loss, val_acc = train_and_validate(folder_path)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training model...\n",
            "1. Now training on AP1295.txt\n",
            "80/80 - 10s - loss: 0.3516 - accuracy: 0.8623 - val_loss: 0.5217 - val_accuracy: 0.8322\n",
            "2. Now training on AHG0050.txt\n",
            "148/148 - 8s - loss: 0.4359 - accuracy: 0.8463 - val_loss: 0.0090 - val_accuracy: 1.0000\n",
            "3. Now training on AP1291.txt\n",
            "111/111 - 8s - loss: 0.4116 - accuracy: 0.8398 - val_loss: 5.2620 - val_accuracy: 0.0000e+00\n",
            "4. Now training on AP1262.txt\n",
            "41/41 - 7s - loss: 0.5327 - accuracy: 0.8252 - val_loss: 0.0094 - val_accuracy: 1.0000\n",
            "5. Now training on AP1194.txt\n",
            "161/161 - 9s - loss: 0.4253 - accuracy: 0.8277 - val_loss: 4.9662 - val_accuracy: 0.0000e+00\n",
            "6. Now training on AP1219.txt\n",
            "113/113 - 8s - loss: 0.2159 - accuracy: 0.9088 - val_loss: 4.0614 - val_accuracy: 0.0000e+00\n",
            "7. Now training on AP1127.txt\n",
            "94/94 - 8s - loss: 0.3488 - accuracy: 0.8588 - val_loss: 2.9217 - val_accuracy: 0.2889\n",
            "8. Now training on AP1160.txt\n",
            "198/198 - 10s - loss: 0.2990 - accuracy: 0.8680 - val_loss: 7.3138 - val_accuracy: 0.0000e+00\n",
            "9. Now training on AP1006.txt\n",
            "126/126 - 8s - loss: 0.2598 - accuracy: 0.9028 - val_loss: 5.3799e-05 - val_accuracy: 1.0000\n",
            "10. Now training on AP1205.txt\n",
            "139/139 - 1s - loss: 0.2212 - accuracy: 0.9103 - val_loss: 7.4452 - val_accuracy: 0.1098\n",
            "11. Now training on AP1035.txt\n",
            "130/130 - 8s - loss: 0.7198 - accuracy: 0.6531 - val_loss: 3.5000 - val_accuracy: 0.0000e+00\n",
            "12. Now training on AP1247.txt\n",
            "66/66 - 9s - loss: 0.1369 - accuracy: 0.9623 - val_loss: 5.4408 - val_accuracy: 0.0000e+00\n",
            "13. Now training on AP1264.txt\n",
            "121/121 - 8s - loss: 0.2406 - accuracy: 0.8833 - val_loss: 0.0130 - val_accuracy: 1.0000\n",
            "14. Now training on AP1150.txt\n",
            "160/160 - 9s - loss: 0.3998 - accuracy: 0.8401 - val_loss: 2.4621 - val_accuracy: 0.5203\n",
            "15. Now training on AP1231.txt\n",
            "120/120 - 10s - loss: 0.2010 - accuracy: 0.9220 - val_loss: 4.8997 - val_accuracy: 0.0000e+00\n",
            "16. Now training on AP1232.txt\n",
            "141/141 - 10s - loss: 0.2732 - accuracy: 0.8867 - val_loss: 5.1446 - val_accuracy: 0.0000e+00\n",
            "17. Now training on AHG0071.txt\n",
            "138/138 - 5s - loss: 0.2334 - accuracy: 0.8659 - val_loss: 6.5280 - val_accuracy: 0.0000e+00\n",
            "18. Now training on AP1241.txt\n",
            "106/106 - 8s - loss: 0.3971 - accuracy: 0.8303 - val_loss: 6.3698 - val_accuracy: 0.0000e+00\n",
            "19. Now training on AHG0056.txt\n",
            "182/182 - 5s - loss: 0.4059 - accuracy: 0.8099 - val_loss: 0.9172 - val_accuracy: 0.4826\n",
            "20. Now training on AP1168.txt\n",
            "143/143 - 7s - loss: 0.4215 - accuracy: 0.7819 - val_loss: 2.0153 - val_accuracy: 0.4241\n",
            "21. Now training on AHG0050.txt\n",
            "148/148 - 2s - loss: 0.2415 - accuracy: 0.8823 - val_loss: 0.0020 - val_accuracy: 1.0000\n",
            "22. Now training on AP1094.txt\n",
            "104/104 - 8s - loss: 0.1327 - accuracy: 0.9616 - val_loss: 5.6092 - val_accuracy: 0.1403\n",
            "23. Now training on AHG0140.txt\n",
            "135/135 - 2s - loss: 0.3414 - accuracy: 0.8499 - val_loss: 9.2462e-05 - val_accuracy: 1.0000\n",
            "24. Now training on AP1094.txt\n",
            "104/104 - 1s - loss: 0.0282 - accuracy: 0.9922 - val_loss: 2.7008 - val_accuracy: 0.3072\n",
            "25. Now training on AP1108.txt\n",
            "132/132 - 10s - loss: 0.4825 - accuracy: 0.8191 - val_loss: 6.5796 - val_accuracy: 0.0000e+00\n",
            "26. Now training on AP1115.txt\n",
            "116/116 - 5s - loss: 0.2997 - accuracy: 0.8619 - val_loss: 1.1159 - val_accuracy: 0.8277\n",
            "27. Now training on AP1249.txt\n",
            "155/155 - 9s - loss: 0.5784 - accuracy: 0.7388 - val_loss: 0.3901 - val_accuracy: 0.8938\n",
            "28. Now training on AP1019.txt\n",
            "77/77 - 8s - loss: 0.3253 - accuracy: 0.8799 - val_loss: 0.5900 - val_accuracy: 1.0000\n",
            "29. Now training on AP1011.txt\n",
            "147/147 - 2s - loss: 0.6185 - accuracy: 0.7364 - val_loss: 0.0014 - val_accuracy: 1.0000\n",
            "30. Now training on AP1112.txt\n",
            "119/119 - 5s - loss: 0.5118 - accuracy: 0.8095 - val_loss: 4.1856 - val_accuracy: 0.1778\n",
            "31. Now training on AHG0136.txt\n",
            "148/148 - 9s - loss: 0.5018 - accuracy: 0.7869 - val_loss: 0.4024 - val_accuracy: 0.9296\n",
            "32. Now training on AP1117.txt\n",
            "156/156 - 5s - loss: 0.1619 - accuracy: 0.9313 - val_loss: 11.5926 - val_accuracy: 0.1087\n",
            "33. Now training on AP1216.txt\n",
            "148/148 - 7s - loss: 0.4568 - accuracy: 0.7844 - val_loss: 3.2171 - val_accuracy: 0.0000e+00\n",
            "34. Now training on AHG0036.txt\n",
            "91/91 - 4s - loss: 0.3344 - accuracy: 0.8993 - val_loss: 5.4447 - val_accuracy: 0.0000e+00\n",
            "35. Now training on AP1019.txt\n",
            "77/77 - 1s - loss: 0.3118 - accuracy: 0.8908 - val_loss: 0.6878 - val_accuracy: 0.8799\n",
            "36. Now training on AHG0134.txt\n",
            "131/131 - 1s - loss: 0.2528 - accuracy: 0.9097 - val_loss: 1.3603e-04 - val_accuracy: 1.0000\n",
            "37. Now training on AP1270.txt\n",
            "185/185 - 5s - loss: 0.3195 - accuracy: 0.8565 - val_loss: 5.2500 - val_accuracy: 0.0000e+00\n",
            "38. Now training on AP1198.txt\n",
            "129/129 - 6s - loss: 0.1985 - accuracy: 0.9048 - val_loss: 4.7483 - val_accuracy: 0.1811\n",
            "39. Now training on AP1180.txt\n",
            "211/211 - 2s - loss: 0.4013 - accuracy: 0.8424 - val_loss: 4.3219 - val_accuracy: 0.4273\n",
            "40. Now training on AHG0016.txt\n",
            "162/162 - 2s - loss: 0.5304 - accuracy: 0.7195 - val_loss: 7.2054 - val_accuracy: 0.2036\n",
            "41. Now training on AHG0057.txt\n",
            "130/130 - 5s - loss: 0.2900 - accuracy: 0.8727 - val_loss: 6.1070 - val_accuracy: 0.2407\n",
            "42. Now training on AP1179.txt\n",
            "195/195 - 6s - loss: 0.3838 - accuracy: 0.8210 - val_loss: 0.3811 - val_accuracy: 0.9234\n",
            "43. Now training on AP1151.txt\n",
            "100/100 - 5s - loss: 0.1796 - accuracy: 0.9181 - val_loss: 5.5443 - val_accuracy: 0.0000e+00\n",
            "44. Now training on AP1140.txt\n",
            "67/67 - 1s - loss: 0.6043 - accuracy: 0.7336 - val_loss: 0.1022 - val_accuracy: 1.0000\n",
            "45. Now training on AHG0053.txt\n",
            "184/184 - 10s - loss: 0.1723 - accuracy: 0.9343 - val_loss: 5.7057 - val_accuracy: 0.0890\n",
            "46. Now training on AP1026.txt\n",
            "101/101 - 12s - loss: 0.3711 - accuracy: 0.8328 - val_loss: 6.0039e-05 - val_accuracy: 1.0000\n",
            "47. Now training on AP1234.txt\n",
            "143/143 - 5s - loss: 0.2845 - accuracy: 0.8895 - val_loss: 4.2580 - val_accuracy: 0.0000e+00\n",
            "48. Now training on AP1294.txt\n",
            "139/139 - 5s - loss: 0.2276 - accuracy: 0.8934 - val_loss: 6.3566 - val_accuracy: 0.0565\n",
            "49. Now training on AP1263.txt\n",
            "104/104 - 1s - loss: 0.2198 - accuracy: 0.9130 - val_loss: 4.2851 - val_accuracy: 0.0000e+00\n",
            "50. Now training on AHG0070.txt\n",
            "202/202 - 12s - loss: 0.4965 - accuracy: 0.7865 - val_loss: 6.2546 - val_accuracy: 0.0878\n",
            "51. Now training on AP1061.txt\n",
            "126/126 - 8s - loss: 0.1297 - accuracy: 0.9599 - val_loss: 1.3508e-04 - val_accuracy: 1.0000\n",
            "52. Now training on AP1221.txt\n",
            "132/132 - 5s - loss: 0.3279 - accuracy: 0.8642 - val_loss: 1.8146 - val_accuracy: 0.5515\n",
            "53. Now training on AP1128.txt\n",
            "81/81 - 4s - loss: 0.3840 - accuracy: 0.8445 - val_loss: 5.1112 - val_accuracy: 0.0000e+00\n",
            "54. Now training on AP1171.txt\n",
            "126/126 - 8s - loss: 0.3764 - accuracy: 0.8399 - val_loss: 4.4141e-04 - val_accuracy: 1.0000\n",
            "55. Now training on AP1158.txt\n",
            "151/151 - 2s - loss: 0.1989 - accuracy: 0.9104 - val_loss: 3.1511 - val_accuracy: 0.3866\n",
            "56. Now training on AP1298.txt\n",
            "167/167 - 2s - loss: 0.2553 - accuracy: 0.8961 - val_loss: 3.3738e-06 - val_accuracy: 1.0000\n",
            "57. Now training on AP1168.txt\n",
            "143/143 - 1s - loss: 0.4307 - accuracy: 0.7841 - val_loss: 2.1933 - val_accuracy: 0.3847\n",
            "58. Now training on AP1254.txt\n",
            "123/123 - 6s - loss: 0.2823 - accuracy: 0.8621 - val_loss: 2.8901 - val_accuracy: 0.5235\n",
            "59. Now training on AP1276.txt\n",
            "125/125 - 10s - loss: 0.2740 - accuracy: 0.8776 - val_loss: 0.2458 - val_accuracy: 1.0000\n",
            "60. Now training on AHG0047.txt\n",
            "152/152 - 2s - loss: 0.5263 - accuracy: 0.8023 - val_loss: 0.3616 - val_accuracy: 0.9054\n",
            "61. Now training on AHG0131.txt\n",
            "101/101 - 1s - loss: 0.1263 - accuracy: 0.9540 - val_loss: 4.2002 - val_accuracy: 0.0000e+00\n",
            "62. Now training on AP1141.txt\n",
            "125/125 - 1s - loss: 0.2171 - accuracy: 0.9206 - val_loss: 5.0014 - val_accuracy: 0.2282\n",
            "63. Now training on AP1056.txt\n",
            "108/108 - 5s - loss: 0.1668 - accuracy: 0.9497 - val_loss: 1.6194e-05 - val_accuracy: 1.0000\n",
            "64. Now training on AP1195.txt\n",
            "87/87 - 4s - loss: 0.3906 - accuracy: 0.8837 - val_loss: 1.4182 - val_accuracy: 0.5995\n",
            "65. Now training on AHG0008.txt\n",
            "185/185 - 2s - loss: 0.6085 - accuracy: 0.7421 - val_loss: 0.9124 - val_accuracy: 0.5348\n",
            "66. Now training on AP1074.txt\n",
            "87/87 - 4s - loss: 0.2001 - accuracy: 0.9587 - val_loss: 0.0014 - val_accuracy: 1.0000\n",
            "67. Now training on AHG0071.txt\n",
            "138/138 - 1s - loss: 0.2827 - accuracy: 0.8610 - val_loss: 5.2728 - val_accuracy: 0.0000e+00\n",
            "68. Now training on AP1254.txt\n",
            "123/123 - 1s - loss: 0.3536 - accuracy: 0.8531 - val_loss: 4.7293 - val_accuracy: 0.0000e+00\n",
            "69. Now training on AP1077.txt\n",
            "169/169 - 5s - loss: 0.4521 - accuracy: 0.7758 - val_loss: 6.7085 - val_accuracy: 0.1646\n",
            "70. Now training on AP1030.txt\n",
            "116/116 - 5s - loss: 0.2292 - accuracy: 0.9295 - val_loss: 6.1811e-05 - val_accuracy: 1.0000\n",
            "71. Now training on AP1180.txt\n",
            "211/211 - 2s - loss: 0.4725 - accuracy: 0.8198 - val_loss: 3.6311 - val_accuracy: 0.4273\n",
            "72. Now training on AP1211.txt\n",
            "94/94 - 1s - loss: 0.2488 - accuracy: 0.8772 - val_loss: 4.5653 - val_accuracy: 0.0000e+00\n",
            "73. Now training on AP1082.txt\n",
            "115/115 - 8s - loss: 0.3018 - accuracy: 0.8660 - val_loss: 0.8936 - val_accuracy: 0.7256\n",
            "74. Now training on AHG0082.txt\n",
            "119/119 - 1s - loss: 0.3222 - accuracy: 0.8506 - val_loss: 0.9406 - val_accuracy: 0.7488\n",
            "75. Now training on AP1149.txt\n",
            "58/58 - 4s - loss: 0.3112 - accuracy: 0.9221 - val_loss: 2.8940e-05 - val_accuracy: 1.0000\n",
            "76. Now training on AP1154.txt\n",
            "178/178 - 5s - loss: 0.7312 - accuracy: 0.7520 - val_loss: 7.0893 - val_accuracy: 0.2354\n",
            "77. Now training on AP1293.txt\n",
            "119/119 - 1s - loss: 0.2828 - accuracy: 0.9054 - val_loss: 0.0037 - val_accuracy: 1.0000\n",
            "78. Now training on AHG0090.txt\n",
            "184/184 - 2s - loss: 0.2663 - accuracy: 0.8777 - val_loss: 0.1924 - val_accuracy: 0.9383\n",
            "79. Now training on AHG0002.txt\n",
            "99/99 - 10s - loss: 0.3129 - accuracy: 0.8624 - val_loss: 5.4955 - val_accuracy: 0.1003\n",
            "80. Now training on AP1227.txt\n",
            "146/146 - 5s - loss: 0.3185 - accuracy: 0.8490 - val_loss: 9.5600 - val_accuracy: 0.0492\n",
            "81. Now training on AP1207.txt\n",
            "98/98 - 5s - loss: 0.1374 - accuracy: 0.9744 - val_loss: 9.6065 - val_accuracy: 0.0464\n",
            "82. Now training on AP1069.txt\n",
            "68/68 - 1s - loss: 0.4985 - accuracy: 0.7982 - val_loss: 2.1117 - val_accuracy: 0.0000e+00\n",
            "83. Now training on AP1137.txt\n",
            "144/144 - 1s - loss: 0.5255 - accuracy: 0.8010 - val_loss: 4.7305 - val_accuracy: 0.0000e+00\n",
            "84. Now training on AP1148.txt\n",
            "166/166 - 2s - loss: 0.2474 - accuracy: 0.8838 - val_loss: 6.9827 - val_accuracy: 0.3277\n",
            "85. Now training on AP1172.txt\n",
            "146/146 - 5s - loss: 0.3211 - accuracy: 0.8661 - val_loss: 3.9848 - val_accuracy: 0.2528\n",
            "86. Now training on AHG0133.txt\n",
            "161/161 - 7s - loss: 0.2755 - accuracy: 0.8679 - val_loss: 5.0620 - val_accuracy: 0.0000e+00\n",
            "87. Now training on AHG0106.txt\n",
            "157/157 - 5s - loss: 0.3120 - accuracy: 0.8698 - val_loss: 0.0057 - val_accuracy: 1.0000\n",
            "88. Now training on AP1208.txt\n",
            "174/174 - 2s - loss: 0.4531 - accuracy: 0.8043 - val_loss: 0.2993 - val_accuracy: 0.8325\n",
            "89. Now training on AP1175.txt\n",
            "113/113 - 6s - loss: 0.4037 - accuracy: 0.8046 - val_loss: 3.3794 - val_accuracy: 0.3335\n",
            "90. Now training on AP1302.txt\n",
            "194/194 - 6s - loss: 0.4120 - accuracy: 0.8068 - val_loss: 4.8841 - val_accuracy: 0.0580\n",
            "91. Now training on AP1040.txt\n",
            "121/121 - 5s - loss: 0.3224 - accuracy: 0.8642 - val_loss: 3.6868 - val_accuracy: 0.0000e+00\n",
            "92. Now training on AHG0044.txt\n",
            "201/201 - 2s - loss: 0.4350 - accuracy: 0.8239 - val_loss: 0.3336 - val_accuracy: 0.7773\n",
            "93. Now training on AP1247.txt\n",
            "66/66 - 1s - loss: 0.2044 - accuracy: 0.9559 - val_loss: 9.7728 - val_accuracy: 0.0000e+00\n",
            "94. Now training on AP1126.txt\n",
            "138/138 - 1s - loss: 0.5472 - accuracy: 0.7727 - val_loss: 0.0047 - val_accuracy: 1.0000\n",
            "95. Now training on AHG0047.txt\n",
            "152/152 - 2s - loss: 0.4507 - accuracy: 0.8158 - val_loss: 1.3556 - val_accuracy: 0.5220\n",
            "96. Now training on AP1047.txt\n",
            "135/135 - 5s - loss: 0.4272 - accuracy: 0.7825 - val_loss: 1.1729 - val_accuracy: 0.7150\n",
            "97. Now training on AP1218.txt\n",
            "113/113 - 5s - loss: 0.4020 - accuracy: 0.7777 - val_loss: 0.1524 - val_accuracy: 0.9744\n",
            "98. Now training on AP1280.txt\n",
            "127/127 - 12s - loss: 0.3112 - accuracy: 0.8526 - val_loss: 1.9330 - val_accuracy: 0.6094\n",
            "99. Now training on AP1231.txt\n",
            "120/120 - 1s - loss: 0.1878 - accuracy: 0.9256 - val_loss: 3.4102 - val_accuracy: 0.1423\n",
            "100. Now training on AP1031.txt\n",
            "133/133 - 1s - loss: 0.0337 - accuracy: 0.9922 - val_loss: 1.8302e-07 - val_accuracy: 1.0000\n",
            "101. Now training on AP1169.txt\n",
            "128/128 - 5s - loss: 0.4414 - accuracy: 0.8145 - val_loss: 1.3739 - val_accuracy: 0.7926\n",
            "102. Now training on AP1091.txt\n",
            "109/109 - 1s - loss: 0.3653 - accuracy: 0.8307 - val_loss: 2.5610 - val_accuracy: 0.0884\n",
            "103. Now training on AHG0070.txt\n",
            "202/202 - 2s - loss: 0.4340 - accuracy: 0.8058 - val_loss: 6.2047 - val_accuracy: 0.0605\n",
            "104. Now training on AHG0044.txt\n",
            "201/201 - 2s - loss: 0.3608 - accuracy: 0.8540 - val_loss: 0.2121 - val_accuracy: 0.8950\n",
            "105. Now training on AP1165.txt\n",
            "113/113 - 5s - loss: 0.2742 - accuracy: 0.9130 - val_loss: 9.0008 - val_accuracy: 0.1706\n",
            "106. Now training on AP1269.txt\n",
            "136/136 - 8s - loss: 0.4794 - accuracy: 0.8064 - val_loss: 0.3763 - val_accuracy: 1.0000\n",
            "107. Now training on AP1103.txt\n",
            "82/82 - 4s - loss: 0.3117 - accuracy: 0.8603 - val_loss: 1.6668 - val_accuracy: 0.1608\n",
            "108. Now training on AP1131.txt\n",
            "68/68 - 1s - loss: 0.3913 - accuracy: 0.8837 - val_loss: 0.1248 - val_accuracy: 1.0000\n",
            "109. Now training on AP1254.txt\n",
            "123/123 - 1s - loss: 0.3460 - accuracy: 0.8551 - val_loss: 4.9977 - val_accuracy: 0.0000e+00\n",
            "110. Now training on AP1152.txt\n",
            "132/132 - 1s - loss: 0.2339 - accuracy: 0.8838 - val_loss: 7.0361 - val_accuracy: 0.4247\n",
            "111. Now training on AHG0045.txt\n",
            "168/168 - 2s - loss: 0.3212 - accuracy: 0.8551 - val_loss: 4.5423 - val_accuracy: 0.4377\n",
            "112. Now training on AP1131.txt\n",
            "68/68 - 1s - loss: 0.3206 - accuracy: 0.9046 - val_loss: 3.3888 - val_accuracy: 0.3479\n",
            "113. Now training on AP1205.txt\n",
            "139/139 - 1s - loss: 0.2201 - accuracy: 0.9223 - val_loss: 4.9050 - val_accuracy: 0.1098\n",
            "114. Now training on AP1174.txt\n",
            "98/98 - 1s - loss: 0.3792 - accuracy: 0.8423 - val_loss: 2.6328e-04 - val_accuracy: 1.0000\n",
            "115. Now training on AP1010.txt\n",
            "149/149 - 2s - loss: 0.3513 - accuracy: 0.8713 - val_loss: 8.4418 - val_accuracy: 0.0000e+00\n",
            "116. Now training on AP1136.txt\n",
            "173/173 - 5s - loss: 0.2956 - accuracy: 0.8813 - val_loss: 6.1030 - val_accuracy: 0.1074\n",
            "117. Now training on AP1214.txt\n",
            "125/125 - 1s - loss: 0.2316 - accuracy: 0.8941 - val_loss: 1.3408e-04 - val_accuracy: 1.0000\n",
            "118. Now training on AP1205.txt\n",
            "139/139 - 1s - loss: 0.1862 - accuracy: 0.9215 - val_loss: 7.1823 - val_accuracy: 0.2120\n",
            "119. Now training on AP1205.txt\n",
            "139/139 - 1s - loss: 0.1278 - accuracy: 0.9372 - val_loss: 9.2462 - val_accuracy: 0.1959\n",
            "120. Now training on AHG0071.txt\n",
            "138/138 - 1s - loss: 0.2170 - accuracy: 0.8824 - val_loss: 5.7783 - val_accuracy: 0.0000e+00\n",
            "121. Now training on AP1029.txt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-12-350a418aa658>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m \u001b[0mfolder_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"/content/processed/\"\u001b[0m \u001b[0;31m# The only path that I have to change for NCI\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m \u001b[0mtrain_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_acc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_and_validate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfolder_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-12-350a418aa658>\u001b[0m in \u001b[0;36mtrain_and_validate\u001b[0;34m(input_folder_path)\u001b[0m\n\u001b[1;32m     31\u001b[0m                         \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m                         \u001b[0mshuffle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;31m# shuffle the sequences, but not the individual data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m                         validation_split = 0.1, verbose = 2)\n\u001b[0m\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0mtrain_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     64\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m     \u001b[0;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m    846\u001b[0m                 batch_size=batch_size):\n\u001b[1;32m    847\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 848\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    849\u001b[0m               \u001b[0;31m# Catch OutOfRangeError for Datasets of unknown size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    850\u001b[0m               \u001b[0;31m# This blocks until the batch has finished executing.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    578\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    579\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 580\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    581\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    582\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    609\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    610\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 611\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    612\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    613\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2418\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2419\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2420\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2422\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1663\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1664\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1665\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1667\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1744\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1746\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    596\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    597\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    599\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PXnLJnjHv24h",
        "colab_type": "text"
      },
      "source": [
        "The code below trains on individual data - not really recommended?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQYlKEGp0_AV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Case where data is shuffled\n",
        "'''\n",
        "look_back = SEQ_LENGTH\n",
        "NO_OF_SAMPLES = 100\n",
        "\n",
        "def train_and_validate(input_folder_path):\n",
        "    \n",
        "    print(\"Training model...\")\n",
        "    train_loss = []\n",
        "    train_acc = []\n",
        "    val_loss = []\n",
        "    val_acc = []\n",
        "\n",
        "    folder = os.listdir(input_folder_path)\n",
        "    # print(len(folder))\n",
        "    # counter = 1\n",
        "\n",
        "    for counter in range(0, NO_OF_SAMPLES):\n",
        "        i = random.randint(0, len(folder)-1) # random might be bad?\n",
        "        input_file_path = input_folder_path+folder[i]\n",
        "        print(f\"{counter+1}. Now training on {folder[i]}\")\n",
        "    # for singleFile in folder:\n",
        "    #     print(f\"Heart No.{counter} - {singleFile}\")\n",
        "    #     input_file_path = input_folder_path + singleFile   \n",
        "        X_train, y_train = load_data_single(input_file_path)\n",
        "        # print(f\"{counter}. Currently retrieving data from {singleFile}\")\n",
        "        X_train, y_train = create_dataset(X_train, y_train, look_back)\n",
        "\n",
        "\n",
        "       # 0.1 rows are used for validation\n",
        "        hist = model.fit(X_train, y_train, \n",
        "                        batch_size=16, epochs=1, \n",
        "                        shuffle = True, # shuffle the sequences, but not the individual data\n",
        "                        validation_split = 0.1, verbose = 1)\n",
        "\n",
        "        train_loss.append(hist.history['loss'][0])\n",
        "        train_acc.append(hist.history['accuracy'][0])\n",
        "        val_loss.append(hist.history['val_loss'][0])\n",
        "        val_acc.append(hist.history['val_accuracy'][0])\n",
        "        counter += 1\n",
        "\n",
        "    print(\"Done Training.\")\n",
        "    return train_loss, train_acc, val_loss, val_acc\n",
        "\n",
        "\n",
        "folder_path = \"/content/processed/\" # The only path that I have to change for NCI\n",
        "train_loss, train_acc, val_loss, val_acc = train_and_validate(folder_path)\n",
        "'''"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ay9quqlMe374",
        "colab_type": "text"
      },
      "source": [
        "## Plot Accuracy and Loss vs Number of Samples being trained"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MEXxnm0h83WX",
        "colab_type": "code",
        "outputId": "b1ec206a-7a20-486c-bce2-1c493f9671cf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from matplotlib import style\n",
        "import pandas as pd\n",
        "\n",
        "style.use(\"ggplot\")\n",
        "\n",
        "def create_acc_loss_graph(train_loss, train_acc, test_loss, test_acc):\n",
        "    epoch = list(range(0, NO_OF_SAMPLES, 1))\n",
        "\n",
        "    d = {'epochs': epoch,\n",
        "        'train_acc': train_acc,\n",
        "        'train_loss': train_loss,\n",
        "        'test_acc': val_acc, \n",
        "        'test_loss': val_loss}\n",
        "\n",
        "    df = pd.DataFrame(d)\n",
        "\n",
        "    df['train_acc_avg'] = df['train_acc'].ewm(alpha=.02).mean()  # exponential weighted moving average\n",
        "    df['test_acc_avg'] = df['test_acc'].ewm(alpha=.02).mean()\n",
        "    df['train_loss_avg'] = df['train_loss'].ewm(alpha=.02).mean()\n",
        "    df['test_loss_avg'] = df['test_loss'].ewm(alpha=.02).mean()\n",
        "\n",
        "    # Then plot using pandas:\n",
        "    df.plot(x='epochs', y=['train_acc_avg', 'test_acc_avg'], figsize=(8,4))\n",
        "    plt.xlabel(\"No of samples/epochs\")\n",
        "    plt.ylabel(\"Accuracy\")\n",
        "    df.plot(x='epochs', y=['train_loss_avg', 'test_loss_avg'], figsize=(8,4))\n",
        "    plt.xlabel(\"No of samples/epochs\")\n",
        "    plt.ylabel(\"Loss\")\n",
        "\n",
        "    plt.show()\n",
        "\n",
        "create_acc_loss_graph(train_loss, train_acc, val_loss, val_acc)\n",
        "\n",
        "# export jpeg"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-32-71032b0bb094>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     31\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m \u001b[0mcreate_acc_loss_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_acc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[0;31m# export jpeg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'train_loss' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rm237SLvx-KS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# print(hist.history['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xQzHfV9w7FQR",
        "colab_type": "text"
      },
      "source": [
        "# Testing Model on a Single Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vSVChmRuzczh",
        "colab_type": "code",
        "outputId": "0f5eeb25-6af9-421a-c978-6568c4106be2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# Unseen\n",
        "input_file = \"/content/processed/AP1113.txt\" #bad accuracy\n",
        "\n",
        "X_test, y_test = load_data_single(input_file, \"standard\")\n",
        "X_test, y_test = create_dataset(X_test, y_test, look_back)\n",
        "print(\"Testing on unseen dataset: \\n\")\n",
        "\n",
        "# model.reset_states()\n",
        "loss, acc = model.evaluate(X_test, y_test, batch_size=8, verbose = 1)\n",
        "\n",
        "print('Test loss:', loss)\n",
        "print('Test accuracy:', acc)\n",
        "\n",
        "# Seen\n",
        "input_file = \"/content/processed/AP1173.txt\" # the last thing that was trained\n",
        "\n",
        "X_test, y_test = load_data_single(input_file, \"standard\")\n",
        "X_test, y_test = create_dataset(X_test, y_test, look_back)\n",
        "\n",
        "print(\"\\nTesting on seen dataset: \\n\")\n",
        "# model.reset_states()\n",
        "loss, acc = model.evaluate(X_test, y_test, batch_size=8, verbose = 1)\n",
        "\n",
        "print('Test loss:', loss)\n",
        "print('Test accuracy:', acc)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing on unseen dataset: \n",
            "\n",
            "2552/2552 [==============================] - 19s 7ms/step - loss: 1.1280 - accuracy: 0.6530\n",
            "Test loss: 1.1279670000076294\n",
            "Test accuracy: 0.6529962420463562\n",
            "\n",
            "Testing on seen dataset: \n",
            "\n",
            "2160/2160 [==============================] - 17s 8ms/step - loss: 3.7751 - accuracy: 0.3578\n",
            "Test loss: 3.775066375732422\n",
            "Test accuracy: 0.3577795922756195\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AAfGRXwvpCWY",
        "colab_type": "code",
        "outputId": "6311c3b1-e64b-4afc-965f-a5584fa6a841",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "# input_file = \"/content/processed/AHG0004.txt\" # unseen - terrible accuracy\n",
        "\n",
        "input_file = \"/content/processed/AP1173.txt\" #seen data \n",
        "\n",
        "X_test, y_test= load_data_single(input_file, \"standard\")\n",
        "X_test, y_test = create_dataset(X_test, y_test, look_back)\n",
        "\n",
        "yhat = model.predict_classes(X_test, verbose=1)\n",
        "\n",
        "\n",
        "# yhat is predicted, y_test is real \n",
        "# correctCount = np.sum(np.argmax(y_test, axis =-1) is yhat) # argmax is inverse of to_categorical()\n",
        "y_test = np.argmax(y_test, axis =-1)\n",
        "correctCount = 0\n",
        "\n",
        "for i, j in zip(y_test, yhat):\n",
        "    if i == j:\n",
        "        correctCount += 1\n",
        "\n",
        "print(f\"Accuracy is {round(correctCount/len(y_test), 4)}.\")\n",
        "\n",
        "\n",
        " "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-34-66e3368edd09>:8: Sequential.predict_classes (from tensorflow.python.keras.engine.sequential) is deprecated and will be removed after 2021-01-01.\n",
            "Instructions for updating:\n",
            "Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "540/540 [==============================] - 3s 5ms/step\n",
            "Accuracy is 0.3578.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SJOrH2n8OxOo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}