1_5_evaluation_ECC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#data manipulation\n",
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#Pytorch geometric\n",
    "import torch\n",
    "from torch import Tensor   \n",
    "import torch_geometric\n",
    "from torch_geometric.data import Data\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch_geometric.loader import DataLoader\n",
    "from torch_geometric.nn import NNConv, MessagePassing\n",
    "from torch.nn import Linear, CrossEntropyLoss, Sequential, ReLU, BCELoss \n",
    "import torch_geometric.transforms as T\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch_geometric.nn import global_mean_pool as gap,  global_max_pool as gmp, global_add_pool as gsp\n",
    "\n",
    "\n",
    "#rdkit\n",
    "from rdkit import Chem                      \n",
    "from rdkit.Chem import GetAdjacencyMatrix       \n",
    "from scipy.sparse import coo_matrix\n",
    "from rdkit.Chem import AllChem\n",
    "from rdkit import Chem, DataStructs\n",
    "\n",
    "#matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "#chemprop\n",
    "import chemprop\n",
    "from chemprop.args import TrainArgs, PredictArgs\n",
    "from chemprop.train import cross_validate, run_training, make_predictions\n",
    "\n",
    "#sklearn\n",
    "import sklearn\n",
    "from sklearn.metrics import classification_report, roc_auc_score\n",
    "\n",
    "#shuffle\n",
    "from random import shuffle\n",
    "\n",
    "#for word embeddings\n",
    "import re\n",
    "import gensim\n",
    "from torchtext.vocab import build_vocab_from_iterator\n",
    "\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "\n",
    "#import the pretrained word-embeddings\n",
    "from gensim.models import KeyedVectors\n",
    "wv = KeyedVectors.load(\"wordvectors.kv\", mmap='r')\n",
    "\n",
    "#GPU\n",
    "import gc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utility Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onek_encoding_unk(value, choices):\n",
    "    \"\"\"\n",
    "    Creates a one-hot encoding with an extra category for uncommon values.\n",
    "\n",
    "    :param value: The value for which the encoding should be one.\n",
    "    :param choices: A list of possible values.\n",
    "    :return: A one-hot encoding of the :code:`value` in a list of length :code:`len(choices) + 1`.\n",
    "             If :code:`value` is not in :code:`choices`, then the final element in the encoding is -1.\n",
    "    \"\"\"\n",
    "    encoding = [0] * (len(choices) + 1)\n",
    "    index = choices.index(value) if value in choices else -1\n",
    "    encoding[index] = 1\n",
    "\n",
    "    return encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Featurization_parameters:\n",
    "    \"\"\"\n",
    "    A class holding molecule featurization parameters as attributes.\n",
    "    \"\"\"\n",
    "    def __init__(self) -> None:\n",
    "\n",
    "        # Atom feature sizes\n",
    "        self.MAX_ATOMIC_NUM = 100\n",
    "        #for one-hot-encoding\n",
    "        self.ATOM_FEATURES = {\n",
    "            'atomic_num': list(range(self.MAX_ATOMIC_NUM)),\n",
    "            'degree': [0, 1, 2, 3, 4, 5],\n",
    "            'formal_charge': [-1, -2, 1, 2, 0],\n",
    "            'chiral_tag': [0, 1, 2, 3],\n",
    "            'num_Hs': [0, 1, 2, 3, 4],\n",
    "            'hybridization': [\n",
    "                Chem.rdchem.HybridizationType.SP,\n",
    "                Chem.rdchem.HybridizationType.SP2,\n",
    "                Chem.rdchem.HybridizationType.SP3,\n",
    "                Chem.rdchem.HybridizationType.SP3D,\n",
    "                Chem.rdchem.HybridizationType.SP3D2\n",
    "            ],\n",
    "        }\n",
    "\n",
    "        # Distance feature sizes\n",
    "        self.PATH_DISTANCE_BINS = list(range(10))\n",
    "        self.THREE_D_DISTANCE_MAX = 20\n",
    "        self.THREE_D_DISTANCE_STEP = 1\n",
    "        self.THREE_D_DISTANCE_BINS = list(range(0, self.THREE_D_DISTANCE_MAX + 1, self.THREE_D_DISTANCE_STEP))\n",
    "\n",
    "        # len(choices) + 1 to include room for uncommon values; + 2 at end for IsAromatic and mass\n",
    "        self.ATOM_FDIM = sum(len(choices) + 1 for choices in self.ATOM_FEATURES.values()) + 2\n",
    "        self.EXTRA_ATOM_FDIM = 0\n",
    "        self.BOND_FDIM = 14\n",
    "        self.EXTRA_BOND_FDIM = 0\n",
    "        self.REACTION_MODE = None\n",
    "        self.EXPLICIT_H = False\n",
    "        self.REACTION = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "PARAMS = Featurization_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def atom_features(atom: Chem.rdchem.Atom, functional_groups=None):\n",
    "    \"\"\"\n",
    "    Builds a feature vector for an atom.\n",
    "\n",
    "    :param atom: An RDKit atom.\n",
    "    :param functional_groups: A k-hot vector indicating the functional groups the atom belongs to.\n",
    "    :return: A list containing the atom features.\n",
    "    \"\"\"\n",
    "    if atom is None:\n",
    "        features = [0] * PARAMS.ATOM_FDIM\n",
    "    else:\n",
    "        features = onek_encoding_unk(atom.GetAtomicNum() - 1, PARAMS.ATOM_FEATURES['atomic_num']) + \\\n",
    "            onek_encoding_unk(atom.GetTotalDegree(), PARAMS.ATOM_FEATURES['degree']) + \\\n",
    "            onek_encoding_unk(atom.GetFormalCharge(), PARAMS.ATOM_FEATURES['formal_charge']) + \\\n",
    "            onek_encoding_unk(int(atom.GetChiralTag()), PARAMS.ATOM_FEATURES['chiral_tag']) + \\\n",
    "            onek_encoding_unk(int(atom.GetTotalNumHs()), PARAMS.ATOM_FEATURES['num_Hs']) + \\\n",
    "            onek_encoding_unk(int(atom.GetHybridization()), PARAMS.ATOM_FEATURES['hybridization']) + \\\n",
    "            [1 if atom.GetIsAromatic() else 0] + \\\n",
    "            [atom.GetMass() * 0.01]  # scaled to about the same range as other features\n",
    "        if functional_groups is not None:\n",
    "            features += functional_groups\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bond_features(bond: Chem.rdchem.Bond):\n",
    "    \"\"\"\n",
    "    Builds a feature vector for a bond.\n",
    "\n",
    "    :param bond: An RDKit bond.\n",
    "    :return: A list containing the bond features.\n",
    "    \"\"\"\n",
    "    if bond is None:\n",
    "        fbond = [1] + [0] * (PARAMS.BOND_FDIM - 1)\n",
    "    else:\n",
    "        bt = bond.GetBondType()\n",
    "        fbond = [\n",
    "            0,  # bond is not None\n",
    "            bt == Chem.rdchem.BondType.SINGLE,\n",
    "            bt == Chem.rdchem.BondType.DOUBLE,\n",
    "            bt == Chem.rdchem.BondType.TRIPLE,\n",
    "            bt == Chem.rdchem.BondType.AROMATIC,\n",
    "            (bond.GetIsConjugated() if bt is not None else 0),\n",
    "            (bond.IsInRing() if bt is not None else 0)\n",
    "        ]\n",
    "        fbond += onek_encoding_unk(int(bond.GetStereo()), list(range(6)))\n",
    "    return fbond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "MORGAN_RADIUS = 2\n",
    "MORGAN_NUM_BITS = 2048\n",
    "#a vector representation (1x2048) for molecular feature "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def morgan_binary_features_generator(mol,\n",
    "                                     radius: int = MORGAN_RADIUS,\n",
    "                                     num_bits: int = MORGAN_NUM_BITS):\n",
    "    \"\"\"\n",
    "    Generates a binary Morgan fingerprint for a molecule.\n",
    "    :param mol: A molecule (i.e., either a SMILES or an RDKit molecule).\n",
    "    :param radius: Morgan fingerprint radius.\n",
    "    :param num_bits: Number of bits in Morgan fingerprint.\n",
    "    :return: A 1D numpy array containing the binary Morgan fingerprint.\n",
    "    \"\"\"\n",
    "    mol = Chem.MolFromSmiles(mol) if type(mol) == str else mol\n",
    "    features_vec = AllChem.GetMorganFingerprintAsBitVect(mol, radius, nBits=num_bits)\n",
    "    features = np.zeros((1,))\n",
    "    DataStructs.ConvertToNumpyArray(features_vec, features)\n",
    "\n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data processing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_process(dataset,batch_size):\n",
    "    SMILES = dataset['SMILES']\n",
    "    data_list = []\n",
    "    for smiles in SMILES:\n",
    "        mol = Chem.MolFromSmiles(smiles)     \n",
    "        mol = Chem.AddHs(mol)  \n",
    "\n",
    "        #generate a global vector features (binary Morgan fingerprint) and convert them\n",
    "        mol_feature = torch.tensor(np.array(morgan_binary_features_generator(mol)))\n",
    "\n",
    "        xs = []\n",
    "        for atom in mol.GetAtoms():\n",
    "            x = atom_features(atom)\n",
    "            xs.append(x)\n",
    "            \n",
    "        x = torch.tensor(np.array(xs))\n",
    "        \n",
    "        edge_indices, edge_attrs = [], []\n",
    "        \n",
    "        for bond in mol.GetBonds():\n",
    "            i = bond.GetBeginAtomIdx()\n",
    "            j = bond.GetEndAtomIdx()\n",
    "    \n",
    "            e = bond_features(bond)\n",
    "\n",
    "            edge_indices += [[i,j],[j,i]]\n",
    "            edge_attrs += [e, e]\n",
    "        \n",
    "        edge_index = torch.tensor(edge_indices)\n",
    "        edge_index = edge_index.t().to(torch.long).view(2, -1)\n",
    "        edge_attr = torch.tensor(edge_attrs).view(-1, 14)\n",
    "        \n",
    "        y = torch.tensor(int(dataset.loc[dataset['SMILES'] == smiles,'Activity'])) #response variable y\n",
    "\n",
    "        smi = smiles\n",
    "\n",
    "        # add smiles and num_feature as the attributes\n",
    "        data = Data(x=x, y=y, edge_index=edge_index,edge_attr = edge_attr, smiles=smi, mol_feature=mol_feature)  \n",
    "        data_list.append(data)   # store processed data into the list\n",
    "        \n",
    "    return DataLoader(data_list,batch_size,shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EarlyStopping:\n",
    "    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n",
    "    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt', trace_func=print):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            patience (int): How long to wait after last time validation loss improved.\n",
    "                            Default: 7\n",
    "            verbose (bool): If True, prints a message for each validation loss improvement. \n",
    "                            Default: False\n",
    "            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n",
    "                            Default: 0\n",
    "            path (str): Path for the checkpoint to be saved to.\n",
    "                            Default: 'checkpoint.pt'\n",
    "            trace_func (function): trace print function.\n",
    "                            Default: print            \n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.verbose = verbose\n",
    "        self.counter = 0\n",
    "        self.best_score = None\n",
    "        self.early_stop = False\n",
    "        self.val_loss_min = np.Inf\n",
    "        self.delta = delta\n",
    "        self.path = path\n",
    "        self.trace_func = trace_func\n",
    "    def __call__(self, val_loss, model):\n",
    "\n",
    "        score = -val_loss\n",
    "\n",
    "        if self.best_score is None:\n",
    "            self.best_score = score\n",
    "            self.save_checkpoint(val_loss, model)\n",
    "        elif score < self.best_score + self.delta:\n",
    "            self.counter += 1\n",
    "            self.trace_func(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n",
    "            if self.counter >= self.patience:\n",
    "                self.early_stop = True\n",
    "        else:\n",
    "            self.best_score = score\n",
    "            self.save_checkpoint(val_loss, model)\n",
    "            self.counter = 0\n",
    "\n",
    "    def save_checkpoint(self, val_loss, model):\n",
    "        '''Saves model when validation loss decrease.'''\n",
    "        if self.verbose:\n",
    "            self.trace_func(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n",
    "        torch.save(model.state_dict(), self.path)\n",
    "        self.val_loss_min = val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_weights(m):\n",
    "  '''\n",
    "    Try resetting model weights to avoid\n",
    "    weight leakage.\n",
    "  '''\n",
    "  for layer in m.children():\n",
    "    if hasattr(layer, 'reset_parameters'):\n",
    "        print(f'Reset trainable parameters of layer = {layer}')\n",
    "        layer.reset_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_metrics(metrics):\n",
    "    AUC = [] \n",
    "    precision = []\n",
    "    recall = []\n",
    "    f1_score = []\n",
    "    accuracy = []\n",
    "    for i in metrics:\n",
    "        AUC.append(i[0])\n",
    "        precision.append(i[1]['weighted avg']['precision'])\n",
    "        recall.append(i[1]['weighted avg']['recall'])\n",
    "        f1_score.append(i[1]['weighted avg']['f1-score'])\n",
    "        accuracy.append(i[1]['accuracy'])\n",
    "    \n",
    "    print('AUC:',np.mean(AUC),'+/-',np.std(AUC))\n",
    "    print('Accuracy:',np.mean(accuracy),'+/-',np.std(accuracy))\n",
    "    print('Precision:',np.mean(precision),'+/-',np.std(precision))\n",
    "    print('Recall:',np.mean(recall),'+/-',np.std(recall))\n",
    "    print('F1-score:',np.mean(f1_score),'+/-',np.std(f1_score))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch,train_loader):\n",
    "    \n",
    "    model.train()   \n",
    "    running_loss = 0 \n",
    "    correct = 0\n",
    "    total = 0\n",
    "    criterion = BCELoss()\n",
    "    for batch in train_loader:\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(batch)\n",
    "        label = batch.y.view(-1,1)\n",
    "        loss = criterion(outputs.float(),label.float())\n",
    "        \n",
    "\n",
    "        loss.backward()   # Compute the gradient of loss function \n",
    "        optimizer.step()  # Update parameters based on gradients.\n",
    "        running_loss += loss.item()\n",
    "        # probability that is larger than 0.5, classify as 1 \n",
    "\n",
    "        pred = (outputs >= 0.5).float()\n",
    "\n",
    "        total += label.size(0)\n",
    "        correct += (pred == label).float().sum()\n",
    "        \n",
    "    \n",
    "    loss = running_loss/len(train_loader)\n",
    "    accuracy = 100*correct/total\n",
    "    \n",
    "    train_accuracy.append(accuracy)\n",
    "    train_loss.append(loss)\n",
    "    \n",
    "    if epoch % 10 == 0:\n",
    "        print('Epoch: '+str(int(epoch)))\n",
    "        print('Train Loss: %.3f | Accuracy: %.3f'%(loss,accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(epoch,test_loader):\n",
    "    model.eval()\n",
    "    \n",
    "    running_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    \n",
    "    \n",
    "    with torch.no_grad():\n",
    "        criterion = BCELoss()\n",
    "        for batch in test_loader:\n",
    "        \n",
    "            outputs = model(batch)\n",
    "            label = batch.y.view(-1,1)\n",
    "\n",
    "            loss = criterion(outputs.float(), label.float())    \n",
    "            running_loss += loss.item()\n",
    "            # probability that is larger than 0.5, classify as 1 \n",
    "            pred = (outputs >= 0.5).float()\n",
    "\n",
    "            total += label.size(0)\n",
    "            correct += (pred == label).float().sum()\n",
    "    \n",
    "        loss = running_loss/len(test_loader)\n",
    "        accuracy = 100*correct/total\n",
    "    \n",
    "        test_accuracy.append(accuracy)\n",
    "        test_loss.append(loss)\n",
    "        if epoch % 10 == 0:\n",
    "            print('Test Loss: %.3f | Accuracy: %.3f'%(loss,accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_set as a whole loader\n",
    "def test_metrics(test_loader):\n",
    "    model.eval()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        labels = []\n",
    "        preds = []\n",
    "        for batch in test_loader:\n",
    "            \n",
    "            labels += list(batch.y.view(-1,1).numpy())\n",
    "            preds += list(model(batch).detach().numpy())\n",
    "        \n",
    "        pred_labels = [1 if i > 0.5 else 0 for i in preds]\n",
    "        auc = roc_auc_score(list(labels), list(preds), average='weighted')\n",
    "        report = classification_report(labels, pred_labels,output_dict=True)\n",
    "        return auc, report\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model construction for ECC\n",
    "\n",
    "#### 1. ECC layer to update node(atom) feature vector of a graph(modelcue).\n",
    "#### 2. Aggregate the updated node feature vector to capture global property\n",
    "####     i.e. apply global_add_pool function over the node features \n",
    "#### 3. Then pass the processed features to a fully connected layer for binary classification. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNConv_graph(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "        self.conv1 = NNConv(in_channels=133 ,out_channels=50, nn=Linear(14, 133*50))\n",
    "        \n",
    "        self.linear1 = Linear(50, 10)\n",
    "        self.linear2 = Linear(10, 1)\n",
    "        \n",
    "    def forward(self,data):\n",
    "        x, edge_index, batch_index, mol_feature, edge_attr = data.x, data.edge_index, data.batch, data.mol_feature, data.edge_attr\n",
    "        x = self.conv1(x.double(), edge_index, edge_attr.double())\n",
    "        \n",
    "        x = gsp(x,batch_index)\n",
    "        \n",
    "        x = F.relu(self.linear1(x))\n",
    "        x = torch.sigmoid(self.linear2(x))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model training and evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GraphSAGE model training and evalutation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1 ......\n",
      "Reset trainable parameters of layer = Linear(in_features=14, out_features=6650, bias=True)\n",
      "Reset trainable parameters of layer = Linear(133, 50, bias=False)\n",
      "Reset trainable parameters of layer = NNConv(133, 50, aggr=add, nn=Linear(in_features=14, out_features=6650, bias=True))\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.607 | Accuracy: 78.472\n",
      "Test Loss: 0.548 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.548013).  Saving model ...\n",
      "Validation loss decreased (0.548013 --> 0.517220).  Saving model ...\n",
      "Validation loss decreased (0.517220 --> 0.498044).  Saving model ...\n",
      "Validation loss decreased (0.498044 --> 0.489252).  Saving model ...\n",
      "Validation loss decreased (0.489252 --> 0.479912).  Saving model ...\n",
      "Validation loss decreased (0.479912 --> 0.467204).  Saving model ...\n",
      "Validation loss decreased (0.467204 --> 0.462490).  Saving model ...\n",
      "Validation loss decreased (0.462490 --> 0.453295).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.453295 --> 0.449529).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.457 | Accuracy: 83.333\n",
      "Test Loss: 0.460 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.449529 --> 0.437257).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.437257 --> 0.420429).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.433 | Accuracy: 83.333\n",
      "Test Loss: 0.418 | Accuracy: 83.333\n",
      "Validation loss decreased (0.420429 --> 0.418064).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.418064 --> 0.409862).  Saving model ...\n",
      "Validation loss decreased (0.409862 --> 0.407013).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.407013 --> 0.400016).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 30\n",
      "Train Loss: 0.430 | Accuracy: 83.333\n",
      "Test Loss: 0.394 | Accuracy: 83.333\n",
      "Validation loss decreased (0.400016 --> 0.393514).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.393514 --> 0.387969).  Saving model ...\n",
      "Validation loss decreased (0.387969 --> 0.386606).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.386606 --> 0.374615).  Saving model ...\n",
      "Epoch: 40\n",
      "Train Loss: 0.384 | Accuracy: 83.333\n",
      "Test Loss: 0.421 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "Validation loss decreased (0.374615 --> 0.370006).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 50\n",
      "Train Loss: 0.388 | Accuracy: 84.896\n",
      "Test Loss: 0.408 | Accuracy: 83.333\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.370006 --> 0.345976).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Epoch: 60\n",
      "Train Loss: 0.340 | Accuracy: 87.500\n",
      "Test Loss: 0.348 | Accuracy: 85.417\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "Validation loss decreased (0.345976 --> 0.342802).  Saving model ...\n",
      "Validation loss decreased (0.342802 --> 0.337119).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Epoch: 70\n",
      "Train Loss: 0.357 | Accuracy: 85.069\n",
      "Test Loss: 0.403 | Accuracy: 84.722\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 2 ......\n",
      "Reset trainable parameters of layer = Linear(in_features=14, out_features=6650, bias=True)\n",
      "Reset trainable parameters of layer = Linear(133, 50, bias=False)\n",
      "Reset trainable parameters of layer = NNConv(133, 50, aggr=add, nn=Linear(in_features=14, out_features=6650, bias=True))\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 1.404 | Accuracy: 71.701\n",
      "Test Loss: 0.682 | Accuracy: 65.972\n",
      "Validation loss decreased (inf --> 0.681620).  Saving model ...\n",
      "Validation loss decreased (0.681620 --> 0.377349).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.377349 --> 0.370865).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.370865 --> 0.358502).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.358502 --> 0.357268).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.400 | Accuracy: 84.896\n",
      "Test Loss: 0.383 | Accuracy: 84.722\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 3 ......\n",
      "Reset trainable parameters of layer = Linear(in_features=14, out_features=6650, bias=True)\n",
      "Reset trainable parameters of layer = Linear(133, 50, bias=False)\n",
      "Reset trainable parameters of layer = NNConv(133, 50, aggr=add, nn=Linear(in_features=14, out_features=6650, bias=True))\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.826 | Accuracy: 70.312\n",
      "Test Loss: 0.728 | Accuracy: 82.639\n",
      "Validation loss decreased (inf --> 0.727602).  Saving model ...\n",
      "Validation loss decreased (0.727602 --> 0.535211).  Saving model ...\n",
      "Validation loss decreased (0.535211 --> 0.449371).  Saving model ...\n",
      "Validation loss decreased (0.449371 --> 0.446944).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.446944 --> 0.429324).  Saving model ...\n",
      "Validation loss decreased (0.429324 --> 0.423176).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.423176 --> 0.422464).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.470 | Accuracy: 82.812\n",
      "Test Loss: 0.518 | Accuracy: 74.306\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.422464 --> 0.407135).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.407135 --> 0.395210).  Saving model ...\n",
      "Validation loss decreased (0.395210 --> 0.388887).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.411 | Accuracy: 85.417\n",
      "Test Loss: 0.393 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.388887 --> 0.384606).  Saving model ...\n",
      "Validation loss decreased (0.384606 --> 0.379982).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.379982 --> 0.375447).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 30\n",
      "Train Loss: 0.377 | Accuracy: 86.458\n",
      "Test Loss: 0.372 | Accuracy: 85.417\n",
      "Validation loss decreased (0.375447 --> 0.372213).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.372213 --> 0.363583).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 40\n",
      "Train Loss: 0.365 | Accuracy: 86.806\n",
      "Test Loss: 0.365 | Accuracy: 88.889\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.363583 --> 0.348076).  Saving model ...\n",
      "Validation loss decreased (0.348076 --> 0.346638).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Epoch: 50\n",
      "Train Loss: 0.323 | Accuracy: 88.021\n",
      "Test Loss: 0.350 | Accuracy: 86.806\n",
      "EarlyStopping counter: 7 out of 10\n",
      "Validation loss decreased (0.346638 --> 0.343524).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Epoch: 60\n",
      "Train Loss: 0.312 | Accuracy: 88.542\n",
      "Test Loss: 0.391 | Accuracy: 84.722\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 4 ......\n",
      "Reset trainable parameters of layer = Linear(in_features=14, out_features=6650, bias=True)\n",
      "Reset trainable parameters of layer = Linear(133, 50, bias=False)\n",
      "Reset trainable parameters of layer = NNConv(133, 50, aggr=add, nn=Linear(in_features=14, out_features=6650, bias=True))\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.787 | Accuracy: 75.174\n",
      "Test Loss: 0.727 | Accuracy: 48.611\n",
      "Validation loss decreased (inf --> 0.727126).  Saving model ...\n",
      "Validation loss decreased (0.727126 --> 0.456293).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.456293 --> 0.450639).  Saving model ...\n",
      "Validation loss decreased (0.450639 --> 0.440318).  Saving model ...\n",
      "Validation loss decreased (0.440318 --> 0.418527).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.418527 --> 0.411502).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.420 | Accuracy: 82.118\n",
      "Test Loss: 0.398 | Accuracy: 88.194\n",
      "Validation loss decreased (0.411502 --> 0.398285).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.398285 --> 0.391278).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.391278 --> 0.387823).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.387823 --> 0.380478).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.376 | Accuracy: 84.722\n",
      "Test Loss: 0.377 | Accuracy: 88.889\n",
      "Validation loss decreased (0.380478 --> 0.377356).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.377356 --> 0.365469).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.365469 --> 0.360079).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.388 | Accuracy: 83.854\n",
      "Test Loss: 0.366 | Accuracy: 87.500\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.360079 --> 0.356093).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.356093 --> 0.346535).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 40\n",
      "Train Loss: 0.343 | Accuracy: 87.674\n",
      "Test Loss: 0.341 | Accuracy: 90.278\n",
      "Validation loss decreased (0.346535 --> 0.340918).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.340918 --> 0.337527).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.337527 --> 0.334288).  Saving model ...\n",
      "Validation loss decreased (0.334288 --> 0.323052).  Saving model ...\n",
      "Epoch: 50\n",
      "Train Loss: 0.345 | Accuracy: 87.153\n",
      "Test Loss: 0.371 | Accuracy: 87.500\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.323052 --> 0.321324).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 60\n",
      "Train Loss: 0.315 | Accuracy: 87.153\n",
      "Test Loss: 0.347 | Accuracy: 88.194\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 5 ......\n",
      "Reset trainable parameters of layer = Linear(in_features=14, out_features=6650, bias=True)\n",
      "Reset trainable parameters of layer = Linear(133, 50, bias=False)\n",
      "Reset trainable parameters of layer = NNConv(133, 50, aggr=add, nn=Linear(in_features=14, out_features=6650, bias=True))\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.842 | Accuracy: 76.910\n",
      "Test Loss: 0.538 | Accuracy: 75.694\n",
      "Validation loss decreased (inf --> 0.537508).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.537508 --> 0.517957).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.517957 --> 0.497208).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.497208 --> 0.496659).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.445 | Accuracy: 82.812\n",
      "Test Loss: 0.584 | Accuracy: 82.639\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.496659 --> 0.484249).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.484249 --> 0.478909).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.349 | Accuracy: 86.979\n",
      "Test Loss: 0.531 | Accuracy: 82.639\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.478909 --> 0.471167).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.471167 --> 0.470284).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.449 | Accuracy: 82.465\n",
      "Test Loss: 0.576 | Accuracy: 75.694\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.470284 --> 0.469074).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.469074 --> 0.453546).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40\n",
      "Train Loss: 0.407 | Accuracy: 85.590\n",
      "Test Loss: 0.448 | Accuracy: 82.639\n",
      "Validation loss decreased (0.453546 --> 0.447669).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.447669 --> 0.428338).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 50\n",
      "Train Loss: 0.311 | Accuracy: 88.194\n",
      "Test Loss: 0.422 | Accuracy: 84.028\n",
      "Validation loss decreased (0.428338 --> 0.421970).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.421970 --> 0.394153).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "Epoch: 60\n",
      "Train Loss: 0.342 | Accuracy: 88.542\n",
      "Test Loss: 0.554 | Accuracy: 81.250\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "train_losses = []\n",
    "train_acc = []\n",
    "test_acc = []\n",
    "test_losses= []\n",
    "metrics = []\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    print('Split '+str(i+1)+' ......')\n",
    "    train_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/train_split'+str(i+1)+'.csv')\n",
    "    test_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/test_split'+str(i+1)+'.csv')\n",
    "    train_loader = data_process(train_set,32)\n",
    "    test_loader = data_process(test_set,len(test_set))\n",
    "    \n",
    "    model = NNConv_graph().double()\n",
    "    model.apply(reset_weights)\n",
    "    \n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=0.001,weight_decay=0.001)\n",
    "    \n",
    "    train_loss = []\n",
    "    test_loss = []\n",
    "    train_accuracy = []\n",
    "    test_accuracy = []\n",
    "    \n",
    "    early_stopping = EarlyStopping(patience=10, verbose=True)\n",
    "    #update the model parameters with 50 epochs\n",
    "    for epoch in range(100):\n",
    "        train(epoch,train_loader)\n",
    "        test(epoch,test_loader)\n",
    "        \n",
    "        early_stopping(test_loss[-1],model)\n",
    "        \n",
    "        if early_stopping.early_stop:\n",
    "            print('Early stopping')\n",
    "            break\n",
    "    \n",
    "    model.load_state_dict(torch.load('checkpoint.pt'))\n",
    "    auc, report = test_metrics(test_loader)\n",
    "    metric = [auc,report]\n",
    "    \n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    train_acc.append(train_accuracy)\n",
    "    metrics.append(metric)\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.8209722222222222 +/- 0.026067440948882144\n",
      "Accuracy: 0.8777777777777777 +/- 0.020412414523193145\n",
      "Precision: 0.8661339717571608 +/- 0.02565406421852644\n",
      "Recall: 0.8777777777777777 +/- 0.020412414523193145\n",
      "F1-score: 0.8672254982868836 +/- 0.023653096214976493\n"
     ]
    }
   ],
   "source": [
    "print_metrics(metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}