Ablation_study.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#data manipulation\n",
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#Pytorch geometric\n",
    "import torch   \n",
    "import torch_geometric\n",
    "from torch import nn, Tensor\n",
    "from torch_geometric.data import Data\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch_geometric.loader import DataLoader\n",
    "from torch_geometric.nn import GATConv\n",
    "from torch.nn import Linear, BCELoss, LSTM, Dropout\n",
    "import torch_geometric.transforms as T\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch_geometric.nn import global_mean_pool as gap,  global_max_pool as gmp, global_add_pool as gsp\n",
    "\n",
    "#Pytorch NLP\n",
    "from torch.nn import TransformerEncoderLayer,TransformerEncoder, Embedding\n",
    "\n",
    "#rdkit\n",
    "from rdkit import Chem                      \n",
    "from rdkit.Chem import GetAdjacencyMatrix       \n",
    "from scipy.sparse import coo_matrix\n",
    "from rdkit.Chem import AllChem\n",
    "from rdkit import Chem, DataStructs\n",
    "\n",
    "#matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "#chemprop\n",
    "import chemprop\n",
    "from chemprop.args import TrainArgs, PredictArgs\n",
    "from chemprop.train import cross_validate, run_training, make_predictions\n",
    "\n",
    "#sklearn\n",
    "import sklearn\n",
    "from sklearn.metrics import classification_report, roc_auc_score\n",
    "\n",
    "#shuffle\n",
    "from random import shuffle\n",
    "\n",
    "#for word embeddings\n",
    "import re\n",
    "import gensim\n",
    "from torchtext.vocab import build_vocab_from_iterator\n",
    "\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "\n",
    "#import the pretrained word-embeddings\n",
    "from gensim.models import KeyedVectors\n",
    "wv = KeyedVectors.load(\"dir_to_wordvectors.kv\", mmap='r')\n",
    "\n",
    "#GPU\n",
    "import gc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utility Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onek_encoding_unk(value, choices):\n",
    "    \"\"\"\n",
    "    Creates a one-hot encoding with an extra category for uncommon values.\n",
    "\n",
    "    :param value: The value for which the encoding should be one.\n",
    "    :param choices: A list of possible values.\n",
    "    :return: A one-hot encoding of the :code:`value` in a list of length :code:`len(choices) + 1`.\n",
    "             If :code:`value` is not in :code:`choices`, then the final element in the encoding is -1.\n",
    "    \"\"\"\n",
    "    encoding = [0] * (len(choices) + 1)\n",
    "    index = choices.index(value) if value in choices else -1\n",
    "    encoding[index] = 1\n",
    "\n",
    "    return encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Featurization_parameters:\n",
    "    \"\"\"\n",
    "    A class holding molecule featurization parameters as attributes.\n",
    "    \"\"\"\n",
    "    def __init__(self) -> None:\n",
    "\n",
    "        # Atom feature sizes\n",
    "        self.MAX_ATOMIC_NUM = 100\n",
    "        #for one-hot-encoding\n",
    "        self.ATOM_FEATURES = {\n",
    "            'atomic_num': list(range(self.MAX_ATOMIC_NUM)),\n",
    "            'degree': [0, 1, 2, 3, 4, 5],\n",
    "            'formal_charge': [-1, -2, 1, 2, 0],\n",
    "            'chiral_tag': [0, 1, 2, 3],\n",
    "            'num_Hs': [0, 1, 2, 3, 4],\n",
    "            'hybridization': [\n",
    "                Chem.rdchem.HybridizationType.SP,\n",
    "                Chem.rdchem.HybridizationType.SP2,\n",
    "                Chem.rdchem.HybridizationType.SP3,\n",
    "                Chem.rdchem.HybridizationType.SP3D,\n",
    "                Chem.rdchem.HybridizationType.SP3D2\n",
    "            ],\n",
    "        }\n",
    "\n",
    "        # Distance feature sizes\n",
    "        self.PATH_DISTANCE_BINS = list(range(10))\n",
    "        self.THREE_D_DISTANCE_MAX = 20\n",
    "        self.THREE_D_DISTANCE_STEP = 1\n",
    "        self.THREE_D_DISTANCE_BINS = list(range(0, self.THREE_D_DISTANCE_MAX + 1, self.THREE_D_DISTANCE_STEP))\n",
    "\n",
    "        # len(choices) + 1 to include room for uncommon values; + 2 at end for IsAromatic and mass\n",
    "        self.ATOM_FDIM = sum(len(choices) + 1 for choices in self.ATOM_FEATURES.values()) + 2\n",
    "        self.EXTRA_ATOM_FDIM = 0\n",
    "        self.BOND_FDIM = 14\n",
    "        self.EXTRA_BOND_FDIM = 0\n",
    "        self.REACTION_MODE = None\n",
    "        self.EXPLICIT_H = False\n",
    "        self.REACTION = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "PARAMS = Featurization_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def atom_features(atom: Chem.rdchem.Atom, functional_groups=None):\n",
    "    \"\"\"\n",
    "    Builds a feature vector for an atom.\n",
    "\n",
    "    :param atom: An RDKit atom.\n",
    "    :param functional_groups: A k-hot vector indicating the functional groups the atom belongs to.\n",
    "    :return: A list containing the atom features.\n",
    "    \"\"\"\n",
    "    if atom is None:\n",
    "        features = [0] * PARAMS.ATOM_FDIM\n",
    "    else:\n",
    "        features = onek_encoding_unk(atom.GetAtomicNum() - 1, PARAMS.ATOM_FEATURES['atomic_num']) + \\\n",
    "            onek_encoding_unk(atom.GetTotalDegree(), PARAMS.ATOM_FEATURES['degree']) + \\\n",
    "            onek_encoding_unk(atom.GetFormalCharge(), PARAMS.ATOM_FEATURES['formal_charge']) + \\\n",
    "            onek_encoding_unk(int(atom.GetChiralTag()), PARAMS.ATOM_FEATURES['chiral_tag']) + \\\n",
    "            onek_encoding_unk(int(atom.GetTotalNumHs()), PARAMS.ATOM_FEATURES['num_Hs']) + \\\n",
    "            onek_encoding_unk(int(atom.GetHybridization()), PARAMS.ATOM_FEATURES['hybridization']) + \\\n",
    "            [1 if atom.GetIsAromatic() else 0] + \\\n",
    "            [atom.GetMass() * 0.01]  # scaled to about the same range as other features\n",
    "        if functional_groups is not None:\n",
    "            features += functional_groups\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bond_features(bond: Chem.rdchem.Bond):\n",
    "    \"\"\"\n",
    "    Builds a feature vector for a bond.\n",
    "\n",
    "    :param bond: An RDKit bond.\n",
    "    :return: A list containing the bond features.\n",
    "    \"\"\"\n",
    "    if bond is None:\n",
    "        fbond = [1] + [0] * (PARAMS.BOND_FDIM - 1)\n",
    "    else:\n",
    "        bt = bond.GetBondType()\n",
    "        fbond = [\n",
    "            0,  # bond is not None\n",
    "            bt == Chem.rdchem.BondType.SINGLE,\n",
    "            bt == Chem.rdchem.BondType.DOUBLE,\n",
    "            bt == Chem.rdchem.BondType.TRIPLE,\n",
    "            bt == Chem.rdchem.BondType.AROMATIC,\n",
    "            (bond.GetIsConjugated() if bt is not None else 0),\n",
    "            (bond.IsInRing() if bt is not None else 0)\n",
    "        ]\n",
    "        fbond += onek_encoding_unk(int(bond.GetStereo()), list(range(6)))\n",
    "    return fbond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "MORGAN_RADIUS = 2\n",
    "MORGAN_NUM_BITS = 2048\n",
    "#a vector representation (1x2048) for molecular feature "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def morgan_binary_features_generator(mol,\n",
    "                                     radius: int = MORGAN_RADIUS,\n",
    "                                     num_bits: int = MORGAN_NUM_BITS):\n",
    "    \"\"\"\n",
    "    Generates a binary Morgan fingerprint for a molecule.\n",
    "    :param mol: A molecule (i.e., either a SMILES or an RDKit molecule).\n",
    "    :param radius: Morgan fingerprint radius.\n",
    "    :param num_bits: Number of bits in Morgan fingerprint.\n",
    "    :return: A 1D numpy array containing the binary Morgan fingerprint.\n",
    "    \"\"\"\n",
    "    mol = Chem.MolFromSmiles(mol) if type(mol) == str else mol\n",
    "    features_vec = AllChem.GetMorganFingerprintAsBitVect(mol, radius, nBits=num_bits)\n",
    "    features = np.zeros((1,))\n",
    "    DataStructs.ConvertToNumpyArray(features_vec, features)\n",
    "\n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_tokens(sentences, pad_token = '<pad>'):\n",
    "    \"\"\"\n",
    "    A function that pads a list of token lists to the maximum length with providedpadding token (pad_token) \n",
    "    \"\"\"\n",
    "    max_len = max(len(item) for item in sentences)\n",
    "    padded = [sentence + [pad_token]*(max_len - len(sentence)) for sentence in sentences]\n",
    "    \n",
    "    return padded\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMI_REGEX_PATTERN = r\"\"\"(\\[[^\\]]+]|Br?|Cl?|N|O|S|P|F|I|b|c|n|o|s|p|\\(|\\)|\\.|=|\n",
    "#|-|\\+|\\\\|\\/|:|~|@|\\?|>>?|\\*|\\$|\\%[0-9]{2}|[0-9])\"\"\"\n",
    "\n",
    "class BasicSmilesTokenizer(object):\n",
    "    \"\"\"\n",
    "    Run basic SMILES tokenization using a regex pattern developed by Schwaller et. al. This tokenizer is to be used\n",
    "    when a tokenizer that does not require the transformers library by HuggingFace is required.\n",
    "    Examples\n",
    "    --------\n",
    "    >>> from deepchem.feat.smiles_tokenizer import BasicSmilesTokenizer\n",
    "    >>> tokenizer = BasicSmilesTokenizer()\n",
    "    >>> print(tokenizer.tokenize(\"CC(=O)OC1=CC=CC=C1C(=O)O\"))\n",
    "    ['C', 'C', '(', '=', 'O', ')', 'O', 'C', '1', '=', 'C', 'C', '=', 'C', 'C', '=', 'C', '1', 'C', '(', '=', 'O', ')', 'O']\n",
    "    References\n",
    "    ----------\n",
    "    .. [1]  Philippe Schwaller, Teodoro Laino, Théophile Gaudin, Peter Bolgar, Christopher A. Hunter, Costas Bekas, and Alpha A. Lee\n",
    "            ACS Central Science 2019 5 (9): Molecular Transformer: A Model for Uncertainty-Calibrated Chemical Reaction Prediction\n",
    "            1572-1583 DOI: 10.1021/acscentsci.9b00576\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, regex_pattern: str = SMI_REGEX_PATTERN):\n",
    "        \"\"\" Constructs a BasicSMILESTokenizer.\n",
    "        Parameters\n",
    "        ----------\n",
    "        regex: string\n",
    "            SMILES token regex\n",
    "        \"\"\"\n",
    "        self.regex_pattern = regex_pattern\n",
    "        self.regex = re.compile(self.regex_pattern)\n",
    "\n",
    "    def tokenize(self, text):\n",
    "        \"\"\" Basic Tokenization of a SMILES.\n",
    "        \"\"\"\n",
    "        tokens = [token for token in self.regex.findall(text)]\n",
    "        return tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BasicSmilesTokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def numericalize(smile):\n",
    "    \"\"\"Converts smile into a flat Tensor with indicies of token in the vocab of our pretrained w2v model.\"\"\"\n",
    "    #data = [vocab[item] for item in tokenizer.tokenize(smile)]\n",
    "    data = [wv.get_index(item) for item in tokenizer.tokenize(smile)] \n",
    "    \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "    \"\"\"\n",
    "    A pytroch Module object: \n",
    "    The positional embeddings of Transformer model.\n",
    "    Input:  x: Tensor, shape [seq_len, batch_size, embedding_dim]\n",
    "    \n",
    "    Output: Sum of the original x and positional signal\n",
    "    \"\"\"\n",
    "    def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000):\n",
    "        super().__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "        position = torch.arange(max_len).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))\n",
    "        pe = torch.zeros(max_len, 1, d_model)\n",
    "        pe[:, 0, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 0, 1::2] = torch.cos(position * div_term)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x: Tensor) -> Tensor:\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            x: Tensor, shape [seq_len, batch_size, embedding_dim]\n",
    "        \"\"\"\n",
    "        x = x + self.pe[:x.size(0)]\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_process(dataset,batch_size):\n",
    "    \"\"\"Convert dataset into dataloader with following attributes\n",
    "       x: atom feature matrix of molecule\n",
    "       y: label of molecule\n",
    "       edge_index: [2, num_edges], dtype=torch.Long, matrix that captures connection among atoms in molecule\n",
    "       edge_attr: edge featrue matrix of molecule\n",
    "       smi: a Tensor indicating indicies of tokens in vocab for a SMILES (with paddings)\n",
    "       \n",
    "       Input: A dataframe with a column SMILES, batch_size \n",
    "       \n",
    "       Output: A dataloader with attributes above\n",
    "    \"\"\"\n",
    "    SMILES = dataset['SMILES']\n",
    "    \n",
    "    data_list = []\n",
    "    \n",
    "    for smile in SMILES:\n",
    "        \n",
    "        mol = Chem.MolFromSmiles(smile)     \n",
    "        mol = Chem.AddHs(mol) \n",
    "        \n",
    "        mol_feature = torch.tensor(morgan_binary_features_generator(mol))\n",
    "        \n",
    "        #dtype is numpy, list of lists will be produced in batch\n",
    "        idx_list = numericalize(smile)\n",
    "        \n",
    "        xs = []\n",
    "        for atom in mol.GetAtoms():\n",
    "            x = atom_features(atom)\n",
    "            xs.append(x)\n",
    "            \n",
    "        x = torch.tensor(xs)\n",
    "        \n",
    "        edge_indices, edge_attrs = [], []\n",
    "        \n",
    "        for bond in mol.GetBonds():\n",
    "            i = bond.GetBeginAtomIdx()\n",
    "            j = bond.GetEndAtomIdx()\n",
    "    \n",
    "            e = bond_features(bond)\n",
    "\n",
    "            edge_indices += [[i,j],[j,i]]\n",
    "            edge_attrs += [e, e]\n",
    "        \n",
    "        edge_index = torch.tensor(edge_indices)\n",
    "        edge_index = edge_index.t().to(torch.long).view(2, -1)\n",
    "        edge_attr = torch.tensor(edge_attrs).view(-1, 14)\n",
    "        \n",
    "        y = torch.tensor(int(dataset.loc[dataset['SMILES'] == smile,'Activity'])) #response variable y\n",
    "\n",
    "        # add smiles and num_feature as the attributes\n",
    "        data = Data(x=x, y=y, edge_index=edge_index, edge_attr = edge_attr, emb = idx_list, mol_feature= mol_feature)  \n",
    "        data_list.append(data)   # store processed data into the list\n",
    "        \n",
    "    return DataLoader(data_list,batch_size,shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EarlyStopping:\n",
    "    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n",
    "    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt', trace_func=print):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            patience (int): How long to wait after last time validation loss improved.\n",
    "                            Default: 7\n",
    "            verbose (bool): If True, prints a message for each validation loss improvement. \n",
    "                            Default: False\n",
    "            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n",
    "                            Default: 0\n",
    "            path (str): Path for the checkpoint to be saved to.\n",
    "                            Default: 'checkpoint.pt'\n",
    "            trace_func (function): trace print function.\n",
    "                            Default: print            \n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.verbose = verbose\n",
    "        self.counter = 0\n",
    "        self.best_score = None\n",
    "        self.early_stop = False\n",
    "        self.val_loss_min = np.Inf\n",
    "        self.delta = delta\n",
    "        self.path = path\n",
    "        self.trace_func = trace_func\n",
    "    def __call__(self, val_loss, model):\n",
    "\n",
    "        score = -val_loss\n",
    "\n",
    "        if self.best_score is None:\n",
    "            self.best_score = score\n",
    "            self.save_checkpoint(val_loss, model)\n",
    "        elif score < self.best_score + self.delta:\n",
    "            self.counter += 1\n",
    "            self.trace_func(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n",
    "            if self.counter >= self.patience:\n",
    "                self.early_stop = True\n",
    "        else:\n",
    "            self.best_score = score\n",
    "            self.save_checkpoint(val_loss, model)\n",
    "            self.counter = 0\n",
    "\n",
    "    def save_checkpoint(self, val_loss, model):\n",
    "        '''Saves model when validation loss decrease.'''\n",
    "        if self.verbose:\n",
    "            self.trace_func(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n",
    "        torch.save(model.state_dict(), self.path)\n",
    "        self.val_loss_min = val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_weights(m):\n",
    "  '''\n",
    "    Try resetting model weights to avoid\n",
    "    weight leakage.\n",
    "  '''\n",
    "  for layer in m.children():\n",
    "    if hasattr(layer, 'reset_parameters'):\n",
    "        print(f'Reset trainable parameters of layer = {layer}')\n",
    "        layer.reset_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch,train_loader):\n",
    "    \"\"\"\n",
    "    A function to train model on training set in each epoch.\n",
    "    1. Optimization/\n",
    "    2. Compute and store training losses and accuracies in lists train_loss and train_accuracy, respectively, for each epoch. \n",
    "    3. Report training losses and accuracies every 10 epochs.\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    model.train()   \n",
    "    \n",
    "    running_loss = 0 \n",
    "    correct = 0\n",
    "    total = 0\n",
    "    criterion = BCELoss()\n",
    "    \n",
    "    for batch in train_loader:\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        outputs = model(batch.to(device))\n",
    "        label = batch.y.view(-1,1).to(device)\n",
    "        \n",
    "        loss = criterion(outputs.float(),label.float())\n",
    "        \n",
    "\n",
    "        loss.backward()   # Compute the gradient of loss function \n",
    "        optimizer.step()  # Update parameters based on gradients.\n",
    "        \n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # probability that is larger than 0.5, classify as 1 \n",
    "        pred = (outputs >= 0.5).float()\n",
    "\n",
    "        total += label.size(0)\n",
    "        correct += (pred == label).float().sum()\n",
    "        \n",
    "    #compute losses and accuracies for entire epoch\n",
    "    loss = running_loss/len(train_loader)\n",
    "    accuracy = 100*correct/total\n",
    "    \n",
    "    #to record training losses and accuracies\n",
    "    train_accuracy.append(accuracy)\n",
    "    train_loss.append(loss)\n",
    "    \n",
    "    #report training losses for every 10 epochs\n",
    "    if epoch % 10 == 0:\n",
    "        print('Epoch: '+str(int(epoch)))\n",
    "        print('Train Loss: %.3f | Accuracy: %.3f'%(loss,accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(epoch,test_loader):\n",
    "    \"\"\"\n",
    "    A function to evaluate model on test set in each epoch.\n",
    "    \n",
    "    1. Compute and store test losses and accuracies in lists test_loss and test_accuracy, respectively, for each epoch. \n",
    "    2. Report test losses and accuracies every 10 epochs.\n",
    "    \n",
    "    \"\"\"\n",
    "    model.eval()\n",
    "    \n",
    "    running_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    criterion = BCELoss()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        \n",
    "        for batch in test_loader:            \n",
    "            \n",
    "            outputs = model(batch.to(device))\n",
    "            label = batch.y.view(-1,1).to(device)\n",
    "\n",
    "            loss = criterion(outputs.float(), label.float())    \n",
    "            running_loss += loss.item()\n",
    "            \n",
    "            # probability that is larger than 0.5, classify as 1 \n",
    "            pred = (outputs >= 0.5).float()\n",
    "\n",
    "            total += label.size(0)\n",
    "            correct += (pred == label).float().sum()\n",
    "    \n",
    "        loss = running_loss/len(test_loader)\n",
    "        accuracy = 100*correct/total\n",
    "    \n",
    "        test_accuracy.append(accuracy)\n",
    "        test_loss.append(loss)\n",
    "        if epoch % 10 == 0:\n",
    "            print('Test Loss: %.3f | Accuracy: %.3f'%(loss,accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_set as a whole loader\n",
    "def test_metrics(test_loader):\n",
    "    \"\"\"\n",
    "    A function to evaluate performance metrics of model on test set.\n",
    "    \n",
    "    Compute and return AUC-ROC and classficiation report(Accuracy, weighted average of F1-score, recall and precision ) of the model\n",
    "\n",
    "    \"\"\"\n",
    "    model.eval()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        labels = []\n",
    "        preds = []\n",
    "        \n",
    "        for batch in test_loader:\n",
    "            \n",
    "            labels += list(batch.y.view(-1,1).numpy())\n",
    "            preds += list(model(batch.to(device)).cpu().detach().numpy())\n",
    "        \n",
    "        pred_labels = [1 if i > 0.5 else 0 for i in preds]\n",
    "        \n",
    "        auc = roc_auc_score(list(labels), list(preds), average='weighted')\n",
    "        report = classification_report(labels, pred_labels,output_dict=True)\n",
    "        \n",
    "        return auc, report\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_metrics(metrics):\n",
    "    AUC = [] #0\n",
    "    precision = []\n",
    "    recall = []\n",
    "    f1_score = []\n",
    "    accuracy = []\n",
    "    for i in metrics:\n",
    "        AUC.append(i[0])\n",
    "        precision.append(i[1]['weighted avg']['precision'])\n",
    "        recall.append(i[1]['weighted avg']['recall'])\n",
    "        f1_score.append(i[1]['weighted avg']['f1-score'])\n",
    "        accuracy.append(i[1]['accuracy'])\n",
    "    \n",
    "    print('AUC:',np.mean(AUC),'+/-',np.std(AUC))\n",
    "    print('Accuracy:',np.mean(accuracy),'+/-',np.std(accuracy))\n",
    "    print('Precision:',np.mean(precision),'+/-',np.std(precision))\n",
    "    print('Recall:',np.mean(recall),'+/-',np.std(recall))\n",
    "    print('F1-score:',np.mean(f1_score),'+/-',np.std(f1_score))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combination with 1 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GAT "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GAT(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(GAT, self).__init__()\n",
    "    \n",
    "        #before the Attention Mechanism, parse in a fully connected network and output a 50 dimensional vector \n",
    "        self.hidden = 50  \n",
    "        self.in_head = 5 #repeat the mechanism for 5 times\n",
    "        \n",
    "        self.conv1 = GATConv(in_channels = 133, \n",
    "                             out_channels = self.hidden,\n",
    "                             heads=self.in_head, concat=False) #set concat to be False, so it will take average instead\n",
    "        \n",
    "        self.linear1 = Linear(self.hidden, 100)\n",
    "        self.linear2 = Linear(100, 25)\n",
    "        self.linear3 = Linear(25, 10)\n",
    "        self.linear4 = Linear(10, 1)\n",
    "\n",
    "    def forward(self, data):\n",
    "        \n",
    "        x, edge_index, batch_index = data.x, data.edge_index, data.batch        \n",
    "        #extract batched node and edge feature matrices, edge_index matrix, batch_index and token indices tensor \n",
    "        \n",
    "        x = self.conv1(x.float(), edge_index)\n",
    "\n",
    "        #aggregate the learned local node feature to capture the global property \n",
    "        #here, we apply global_mean_pool over the updated node feature\n",
    "        x = gap(x,batch_index)\n",
    "\n",
    "        # now train a fully connected graph classification network  \n",
    "        x = F.relu(self.linear1(x.float()))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = F.relu(self.linear3(x))\n",
    "        x = torch.sigmoid(self.linear4(x))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.782 | Accuracy: 16.667\n",
      "Test Loss: 0.765 | Accuracy: 16.667\n",
      "Validation loss decreased (inf --> 0.765339).  Saving model ...\n",
      "Validation loss decreased (0.765339 --> 0.714188).  Saving model ...\n",
      "Validation loss decreased (0.714188 --> 0.567170).  Saving model ...\n",
      "Validation loss decreased (0.567170 --> 0.449595).  Saving model ...\n",
      "Validation loss decreased (0.449595 --> 0.448404).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.448404 --> 0.447556).  Saving model ...\n",
      "Validation loss decreased (0.447556 --> 0.447417).  Saving model ...\n",
      "Validation loss decreased (0.447417 --> 0.447327).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.450 | Accuracy: 83.333\n",
      "Test Loss: 0.448 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.447327 --> 0.446839).  Saving model ...\n",
      "Validation loss decreased (0.446839 --> 0.446562).  Saving model ...\n",
      "Validation loss decreased (0.446562 --> 0.446378).  Saving model ...\n",
      "Validation loss decreased (0.446378 --> 0.446315).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.446315 --> 0.445918).  Saving model ...\n",
      "Validation loss decreased (0.445918 --> 0.445904).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.445904 --> 0.445530).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.448 | Accuracy: 83.333\n",
      "Test Loss: 0.445 | Accuracy: 83.333\n",
      "Validation loss decreased (0.445530 --> 0.445486).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.445486 --> 0.445198).  Saving model ...\n",
      "Validation loss decreased (0.445198 --> 0.445078).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.445078 --> 0.444918).  Saving model ...\n",
      "Validation loss decreased (0.444918 --> 0.444805).  Saving model ...\n",
      "Validation loss decreased (0.444805 --> 0.444639).  Saving model ...\n",
      "Validation loss decreased (0.444639 --> 0.444420).  Saving model ...\n",
      "Validation loss decreased (0.444420 --> 0.444321).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.445 | Accuracy: 83.333\n",
      "Test Loss: 0.444 | Accuracy: 83.333\n",
      "Validation loss decreased (0.444321 --> 0.444249).  Saving model ...\n",
      "Validation loss decreased (0.444249 --> 0.444067).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.444067 --> 0.443712).  Saving model ...\n",
      "Validation loss decreased (0.443712 --> 0.443565).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.443565 --> 0.443366).  Saving model ...\n",
      "Validation loss decreased (0.443366 --> 0.443016).  Saving model ...\n",
      "Epoch: 40\n",
      "Train Loss: 0.443 | Accuracy: 83.333\n",
      "Test Loss: 0.443 | Accuracy: 83.333\n",
      "Validation loss decreased (0.443016 --> 0.442919).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.442919 --> 0.442788).  Saving model ...\n",
      "Validation loss decreased (0.442788 --> 0.442510).  Saving model ...\n",
      "Validation loss decreased (0.442510 --> 0.442249).  Saving model ...\n",
      "Validation loss decreased (0.442249 --> 0.442084).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.442084 --> 0.441718).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.441718 --> 0.441515).  Saving model ...\n",
      "Epoch: 50\n",
      "Train Loss: 0.442 | Accuracy: 83.333\n",
      "Test Loss: 0.441 | Accuracy: 83.333\n",
      "Validation loss decreased (0.441515 --> 0.440988).  Saving model ...\n",
      "Validation loss decreased (0.440988 --> 0.440760).  Saving model ...\n",
      "Validation loss decreased (0.440760 --> 0.440410).  Saving model ...\n",
      "Validation loss decreased (0.440410 --> 0.440199).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.440199 --> 0.439987).  Saving model ...\n",
      "Validation loss decreased (0.439987 --> 0.439726).  Saving model ...\n",
      "Validation loss decreased (0.439726 --> 0.438539).  Saving model ...\n",
      "Validation loss decreased (0.438539 --> 0.438300).  Saving model ...\n",
      "Epoch: 60\n",
      "Train Loss: 0.440 | Accuracy: 83.333\n",
      "Test Loss: 0.438 | Accuracy: 83.333\n",
      "Validation loss decreased (0.438300 --> 0.437894).  Saving model ...\n",
      "Validation loss decreased (0.437894 --> 0.437231).  Saving model ...\n",
      "Validation loss decreased (0.437231 --> 0.436659).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.436659 --> 0.435880).  Saving model ...\n",
      "Validation loss decreased (0.435880 --> 0.434930).  Saving model ...\n",
      "Validation loss decreased (0.434930 --> 0.434209).  Saving model ...\n",
      "Validation loss decreased (0.434209 --> 0.433895).  Saving model ...\n",
      "Validation loss decreased (0.433895 --> 0.432901).  Saving model ...\n",
      "Validation loss decreased (0.432901 --> 0.432388).  Saving model ...\n",
      "Epoch: 70\n",
      "Train Loss: 0.431 | Accuracy: 83.333\n",
      "Test Loss: 0.431 | Accuracy: 83.333\n",
      "Validation loss decreased (0.432388 --> 0.431466).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.431466 --> 0.429551).  Saving model ...\n",
      "Validation loss decreased (0.429551 --> 0.428868).  Saving model ...\n",
      "Validation loss decreased (0.428868 --> 0.428759).  Saving model ...\n",
      "Validation loss decreased (0.428759 --> 0.428083).  Saving model ...\n",
      "Validation loss decreased (0.428083 --> 0.427000).  Saving model ...\n",
      "Validation loss decreased (0.427000 --> 0.426330).  Saving model ...\n",
      "Validation loss decreased (0.426330 --> 0.425532).  Saving model ...\n",
      "Epoch: 80\n",
      "Train Loss: 0.429 | Accuracy: 83.160\n",
      "Test Loss: 0.426 | Accuracy: 81.944\n",
      "Validation loss decreased (0.425532 --> 0.425527).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.425527 --> 0.424187).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.424187 --> 0.422629).  Saving model ...\n",
      "Validation loss decreased (0.422629 --> 0.422178).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.422178 --> 0.420117).  Saving model ...\n",
      "Validation loss decreased (0.420117 --> 0.418894).  Saving model ...\n",
      "Epoch: 90\n",
      "Train Loss: 0.422 | Accuracy: 83.160\n",
      "Test Loss: 0.421 | Accuracy: 81.944\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.418894 --> 0.417780).  Saving model ...\n",
      "Validation loss decreased (0.417780 --> 0.416608).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.416608 --> 0.414945).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.414945 --> 0.413777).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.413777 --> 0.411958).  Saving model ...\n",
      "Epoch: 100\n",
      "Train Loss: 0.417 | Accuracy: 82.986\n",
      "Test Loss: 0.412 | Accuracy: 80.556\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.411958 --> 0.410731).  Saving model ...\n",
      "Validation loss decreased (0.410731 --> 0.409806).  Saving model ...\n",
      "Validation loss decreased (0.409806 --> 0.408904).  Saving model ...\n",
      "Validation loss decreased (0.408904 --> 0.408798).  Saving model ...\n",
      "Validation loss decreased (0.408798 --> 0.406407).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.406407 --> 0.405077).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 110\n",
      "Train Loss: 0.409 | Accuracy: 83.333\n",
      "Test Loss: 0.414 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.405077 --> 0.404216).  Saving model ...\n",
      "Validation loss decreased (0.404216 --> 0.400426).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.400426 --> 0.398973).  Saving model ...\n",
      "Validation loss decreased (0.398973 --> 0.397944).  Saving model ...\n",
      "Validation loss decreased (0.397944 --> 0.397247).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.397247 --> 0.396595).  Saving model ...\n",
      "Epoch: 120\n",
      "Train Loss: 0.405 | Accuracy: 83.333\n",
      "Test Loss: 0.404 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.396595 --> 0.393460).  Saving model ...\n",
      "Validation loss decreased (0.393460 --> 0.391552).  Saving model ...\n",
      "Validation loss decreased (0.391552 --> 0.391534).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.391534 --> 0.390481).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.390481 --> 0.389500).  Saving model ...\n",
      "Epoch: 130\n",
      "Train Loss: 0.389 | Accuracy: 83.333\n",
      "Test Loss: 0.389 | Accuracy: 83.333\n",
      "Validation loss decreased (0.389500 --> 0.389301).  Saving model ...\n",
      "Validation loss decreased (0.389301 --> 0.387410).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.387410 --> 0.386144).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 140\n",
      "Train Loss: 0.382 | Accuracy: 83.333\n",
      "Test Loss: 0.384 | Accuracy: 83.333\n",
      "Validation loss decreased (0.386144 --> 0.384388).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.384388 --> 0.383580).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.383580 --> 0.383406).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 150\n",
      "Train Loss: 0.381 | Accuracy: 83.333\n",
      "Test Loss: 0.385 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.383406 --> 0.381437).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.381437 --> 0.380479).  Saving model ...\n",
      "Epoch: 160\n",
      "Train Loss: 0.385 | Accuracy: 83.333\n",
      "Test Loss: 0.386 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.380479 --> 0.380364).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 170\n",
      "Train Loss: 0.384 | Accuracy: 83.333\n",
      "Test Loss: 0.383 | Accuracy: 83.333\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 2 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.748 | Accuracy: 16.667\n",
      "Test Loss: 0.729 | Accuracy: 16.667\n",
      "Validation loss decreased (inf --> 0.729457).  Saving model ...\n",
      "Validation loss decreased (0.729457 --> 0.679433).  Saving model ...\n",
      "Validation loss decreased (0.679433 --> 0.536359).  Saving model ...\n",
      "Validation loss decreased (0.536359 --> 0.450162).  Saving model ...\n",
      "Validation loss decreased (0.450162 --> 0.446878).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.446878 --> 0.446292).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.446292 --> 0.446009).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.450 | Accuracy: 83.333\n",
      "Test Loss: 0.446 | Accuracy: 83.333\n",
      "Validation loss decreased (0.446009 --> 0.445859).  Saving model ...\n",
      "Validation loss decreased (0.445859 --> 0.445466).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.445466 --> 0.444908).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.444908 --> 0.444585).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.444585 --> 0.444495).  Saving model ...\n",
      "Validation loss decreased (0.444495 --> 0.444195).  Saving model ...\n",
      "Validation loss decreased (0.444195 --> 0.444072).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.449 | Accuracy: 83.333\n",
      "Test Loss: 0.444 | Accuracy: 83.333\n",
      "Validation loss decreased (0.444072 --> 0.444037).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.444037 --> 0.444010).  Saving model ...\n",
      "Validation loss decreased (0.444010 --> 0.443639).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.443639 --> 0.443400).  Saving model ...\n",
      "Validation loss decreased (0.443400 --> 0.443293).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.443293 --> 0.443119).  Saving model ...\n",
      "Validation loss decreased (0.443119 --> 0.442968).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.447 | Accuracy: 83.333\n",
      "Test Loss: 0.443 | Accuracy: 83.333\n",
      "Validation loss decreased (0.442968 --> 0.442881).  Saving model ...\n",
      "Validation loss decreased (0.442881 --> 0.442834).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.442834 --> 0.442486).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.442486 --> 0.442265).  Saving model ...\n",
      "Validation loss decreased (0.442265 --> 0.442218).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.442218 --> 0.441863).  Saving model ...\n",
      "Epoch: 40\n",
      "Train Loss: 0.447 | Accuracy: 83.333\n",
      "Test Loss: 0.442 | Accuracy: 83.333\n",
      "Validation loss decreased (0.441863 --> 0.441761).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.441761 --> 0.441732).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.441732 --> 0.441439).  Saving model ...\n",
      "Validation loss decreased (0.441439 --> 0.441103).  Saving model ...\n",
      "Validation loss decreased (0.441103 --> 0.440922).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 50\n",
      "Train Loss: 0.445 | Accuracy: 83.333\n",
      "Test Loss: 0.441 | Accuracy: 83.333\n",
      "Validation loss decreased (0.440922 --> 0.440603).  Saving model ...\n",
      "Validation loss decreased (0.440603 --> 0.440593).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.440593 --> 0.440418).  Saving model ...\n",
      "Validation loss decreased (0.440418 --> 0.440257).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.440257 --> 0.439734).  Saving model ...\n",
      "Validation loss decreased (0.439734 --> 0.439528).  Saving model ...\n",
      "Validation loss decreased (0.439528 --> 0.439402).  Saving model ...\n",
      "Validation loss decreased (0.439402 --> 0.439303).  Saving model ...\n",
      "Epoch: 60\n",
      "Train Loss: 0.445 | Accuracy: 83.333\n",
      "Test Loss: 0.439 | Accuracy: 83.333\n",
      "Validation loss decreased (0.439303 --> 0.439024).  Saving model ...\n",
      "Validation loss decreased (0.439024 --> 0.438987).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.438987 --> 0.438931).  Saving model ...\n",
      "Validation loss decreased (0.438931 --> 0.438508).  Saving model ...\n",
      "Validation loss decreased (0.438508 --> 0.438482).  Saving model ...\n",
      "Validation loss decreased (0.438482 --> 0.437940).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.437940 --> 0.437343).  Saving model ...\n",
      "Validation loss decreased (0.437343 --> 0.437248).  Saving model ...\n",
      "Epoch: 70\n",
      "Train Loss: 0.443 | Accuracy: 83.333\n",
      "Test Loss: 0.437 | Accuracy: 83.333\n",
      "Validation loss decreased (0.437248 --> 0.436973).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.436973 --> 0.436341).  Saving model ...\n",
      "Validation loss decreased (0.436341 --> 0.436106).  Saving model ...\n",
      "Validation loss decreased (0.436106 --> 0.435657).  Saving model ...\n",
      "Validation loss decreased (0.435657 --> 0.435144).  Saving model ...\n",
      "Validation loss decreased (0.435144 --> 0.434842).  Saving model ...\n",
      "Validation loss decreased (0.434842 --> 0.434570).  Saving model ...\n",
      "Validation loss decreased (0.434570 --> 0.433925).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 80\n",
      "Train Loss: 0.443 | Accuracy: 83.333\n",
      "Test Loss: 0.433 | Accuracy: 83.333\n",
      "Validation loss decreased (0.433925 --> 0.433435).  Saving model ...\n",
      "Validation loss decreased (0.433435 --> 0.433166).  Saving model ...\n",
      "Validation loss decreased (0.433166 --> 0.432209).  Saving model ...\n",
      "Validation loss decreased (0.432209 --> 0.431775).  Saving model ...\n",
      "Validation loss decreased (0.431775 --> 0.430934).  Saving model ...\n",
      "Validation loss decreased (0.430934 --> 0.430465).  Saving model ...\n",
      "Validation loss decreased (0.430465 --> 0.430104).  Saving model ...\n",
      "Validation loss decreased (0.430104 --> 0.429391).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.429391 --> 0.427908).  Saving model ...\n",
      "Epoch: 90\n",
      "Train Loss: 0.440 | Accuracy: 83.333\n",
      "Test Loss: 0.427 | Accuracy: 83.333\n",
      "Validation loss decreased (0.427908 --> 0.426584).  Saving model ...\n",
      "Validation loss decreased (0.426584 --> 0.425804).  Saving model ...\n",
      "Validation loss decreased (0.425804 --> 0.425093).  Saving model ...\n",
      "Validation loss decreased (0.425093 --> 0.424185).  Saving model ...\n",
      "Validation loss decreased (0.424185 --> 0.422881).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.422881 --> 0.420930).  Saving model ...\n",
      "Validation loss decreased (0.420930 --> 0.420288).  Saving model ...\n",
      "Validation loss decreased (0.420288 --> 0.419816).  Saving model ...\n",
      "Validation loss decreased (0.419816 --> 0.417877).  Saving model ...\n",
      "Epoch: 100\n",
      "Train Loss: 0.436 | Accuracy: 83.333\n",
      "Test Loss: 0.420 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.417877 --> 0.416260).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.416260 --> 0.414302).  Saving model ...\n",
      "Validation loss decreased (0.414302 --> 0.413810).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.413810 --> 0.411179).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 110\n",
      "Train Loss: 0.427 | Accuracy: 83.160\n",
      "Test Loss: 0.409 | Accuracy: 83.333\n",
      "Validation loss decreased (0.411179 --> 0.408985).  Saving model ...\n",
      "Validation loss decreased (0.408985 --> 0.408189).  Saving model ...\n",
      "Validation loss decreased (0.408189 --> 0.407516).  Saving model ...\n",
      "Validation loss decreased (0.407516 --> 0.406382).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.406382 --> 0.405666).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.405666 --> 0.403189).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.403189 --> 0.401462).  Saving model ...\n",
      "Validation loss decreased (0.401462 --> 0.400029).  Saving model ...\n",
      "Epoch: 120\n",
      "Train Loss: 0.423 | Accuracy: 83.333\n",
      "Test Loss: 0.402 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.400029 --> 0.398200).  Saving model ...\n",
      "Validation loss decreased (0.398200 --> 0.397220).  Saving model ...\n",
      "Validation loss decreased (0.397220 --> 0.396225).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.396225 --> 0.392988).  Saving model ...\n",
      "Validation loss decreased (0.392988 --> 0.391743).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.391743 --> 0.389867).  Saving model ...\n",
      "Epoch: 130\n",
      "Train Loss: 0.417 | Accuracy: 83.333\n",
      "Test Loss: 0.391 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.389867 --> 0.389541).  Saving model ...\n",
      "Validation loss decreased (0.389541 --> 0.387291).  Saving model ...\n",
      "Validation loss decreased (0.387291 --> 0.387099).  Saving model ...\n",
      "Validation loss decreased (0.387099 --> 0.384189).  Saving model ...\n",
      "Validation loss decreased (0.384189 --> 0.382963).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.382963 --> 0.382463).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 140\n",
      "Train Loss: 0.410 | Accuracy: 83.333\n",
      "Test Loss: 0.379 | Accuracy: 83.333\n",
      "Validation loss decreased (0.382463 --> 0.378601).  Saving model ...\n",
      "Validation loss decreased (0.378601 --> 0.378283).  Saving model ...\n",
      "Validation loss decreased (0.378283 --> 0.377537).  Saving model ...\n",
      "Validation loss decreased (0.377537 --> 0.375436).  Saving model ...\n",
      "Validation loss decreased (0.375436 --> 0.375074).  Saving model ...\n",
      "Validation loss decreased (0.375074 --> 0.373251).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.373251 --> 0.372858).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.372858 --> 0.372200).  Saving model ...\n",
      "Epoch: 150\n",
      "Train Loss: 0.407 | Accuracy: 83.333\n",
      "Test Loss: 0.374 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.372200 --> 0.370099).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.370099 --> 0.368233).  Saving model ...\n",
      "Validation loss decreased (0.368233 --> 0.365663).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.365663 --> 0.364565).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 160\n",
      "Train Loss: 0.397 | Accuracy: 83.333\n",
      "Test Loss: 0.369 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.364565 --> 0.364034).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.364034 --> 0.359963).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 170\n",
      "Train Loss: 0.395 | Accuracy: 83.333\n",
      "Test Loss: 0.363 | Accuracy: 83.333\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.359963 --> 0.355007).  Saving model ...\n",
      "Validation loss decreased (0.355007 --> 0.354396).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.354396 --> 0.352866).  Saving model ...\n",
      "Validation loss decreased (0.352866 --> 0.351847).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.351847 --> 0.349761).  Saving model ...\n",
      "Epoch: 180\n",
      "Train Loss: 0.389 | Accuracy: 83.333\n",
      "Test Loss: 0.355 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.349761 --> 0.347753).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.347753 --> 0.346999).  Saving model ...\n",
      "Validation loss decreased (0.346999 --> 0.346700).  Saving model ...\n",
      "Epoch: 190\n",
      "Train Loss: 0.387 | Accuracy: 83.333\n",
      "Test Loss: 0.348 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.346700 --> 0.344558).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.344558 --> 0.343510).  Saving model ...\n",
      "Validation loss decreased (0.343510 --> 0.342892).  Saving model ...\n",
      "Split 3 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.653 | Accuracy: 83.333\n",
      "Test Loss: 0.643 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.642873).  Saving model ...\n",
      "Validation loss decreased (0.642873 --> 0.621836).  Saving model ...\n",
      "Validation loss decreased (0.621836 --> 0.570106).  Saving model ...\n",
      "Validation loss decreased (0.570106 --> 0.457728).  Saving model ...\n",
      "Validation loss decreased (0.457728 --> 0.453337).  Saving model ...\n",
      "Validation loss decreased (0.453337 --> 0.449198).  Saving model ...\n",
      "Validation loss decreased (0.449198 --> 0.448862).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.448862 --> 0.448420).  Saving model ...\n",
      "Validation loss decreased (0.448420 --> 0.448229).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.449 | Accuracy: 83.333\n",
      "Test Loss: 0.448 | Accuracy: 83.333\n",
      "Validation loss decreased (0.448229 --> 0.448000).  Saving model ...\n",
      "Validation loss decreased (0.448000 --> 0.447940).  Saving model ...\n",
      "Validation loss decreased (0.447940 --> 0.447685).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.447685 --> 0.447449).  Saving model ...\n",
      "Validation loss decreased (0.447449 --> 0.447413).  Saving model ...\n",
      "Validation loss decreased (0.447413 --> 0.447272).  Saving model ...\n",
      "Validation loss decreased (0.447272 --> 0.447250).  Saving model ...\n",
      "Validation loss decreased (0.447250 --> 0.447081).  Saving model ...\n",
      "Validation loss decreased (0.447081 --> 0.446797).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.450 | Accuracy: 83.333\n",
      "Test Loss: 0.448 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.446797 --> 0.446698).  Saving model ...\n",
      "Validation loss decreased (0.446698 --> 0.446477).  Saving model ...\n",
      "Validation loss decreased (0.446477 --> 0.446371).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.446371 --> 0.446220).  Saving model ...\n",
      "Validation loss decreased (0.446220 --> 0.446136).  Saving model ...\n",
      "Validation loss decreased (0.446136 --> 0.445939).  Saving model ...\n",
      "Validation loss decreased (0.445939 --> 0.445754).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.448 | Accuracy: 83.333\n",
      "Test Loss: 0.446 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.445754 --> 0.445500).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.445500 --> 0.445350).  Saving model ...\n",
      "Validation loss decreased (0.445350 --> 0.445031).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.445031 --> 0.444934).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 40\n",
      "Train Loss: 0.446 | Accuracy: 83.333\n",
      "Test Loss: 0.445 | Accuracy: 83.333\n",
      "Validation loss decreased (0.444934 --> 0.444639).  Saving model ...\n",
      "Validation loss decreased (0.444639 --> 0.444599).  Saving model ...\n",
      "Validation loss decreased (0.444599 --> 0.444544).  Saving model ...\n",
      "Validation loss decreased (0.444544 --> 0.444316).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.444316 --> 0.444096).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.444096 --> 0.443975).  Saving model ...\n",
      "Validation loss decreased (0.443975 --> 0.443521).  Saving model ...\n",
      "Epoch: 50\n",
      "Train Loss: 0.446 | Accuracy: 83.333\n",
      "Test Loss: 0.444 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.443521 --> 0.443109).  Saving model ...\n",
      "Validation loss decreased (0.443109 --> 0.442975).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.442975 --> 0.442499).  Saving model ...\n",
      "Validation loss decreased (0.442499 --> 0.442294).  Saving model ...\n",
      "Validation loss decreased (0.442294 --> 0.442090).  Saving model ...\n",
      "Validation loss decreased (0.442090 --> 0.441808).  Saving model ...\n",
      "Validation loss decreased (0.441808 --> 0.441590).  Saving model ...\n",
      "Epoch: 60\n",
      "Train Loss: 0.443 | Accuracy: 83.333\n",
      "Test Loss: 0.441 | Accuracy: 83.333\n",
      "Validation loss decreased (0.441590 --> 0.441331).  Saving model ...\n",
      "Validation loss decreased (0.441331 --> 0.441080).  Saving model ...\n",
      "Validation loss decreased (0.441080 --> 0.440645).  Saving model ...\n",
      "Validation loss decreased (0.440645 --> 0.440058).  Saving model ...\n",
      "Validation loss decreased (0.440058 --> 0.439739).  Saving model ...\n",
      "Validation loss decreased (0.439739 --> 0.439332).  Saving model ...\n",
      "Validation loss decreased (0.439332 --> 0.438599).  Saving model ...\n",
      "Validation loss decreased (0.438599 --> 0.437912).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 70\n",
      "Train Loss: 0.440 | Accuracy: 83.333\n",
      "Test Loss: 0.437 | Accuracy: 83.333\n",
      "Validation loss decreased (0.437912 --> 0.437093).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.437093 --> 0.437013).  Saving model ...\n",
      "Validation loss decreased (0.437013 --> 0.433535).  Saving model ...\n",
      "Validation loss decreased (0.433535 --> 0.432510).  Saving model ...\n",
      "Validation loss decreased (0.432510 --> 0.431138).  Saving model ...\n",
      "Validation loss decreased (0.431138 --> 0.429604).  Saving model ...\n",
      "Validation loss decreased (0.429604 --> 0.428455).  Saving model ...\n",
      "Validation loss decreased (0.428455 --> 0.428328).  Saving model ...\n",
      "Validation loss decreased (0.428328 --> 0.427505).  Saving model ...\n",
      "Epoch: 80\n",
      "Train Loss: 0.424 | Accuracy: 83.333\n",
      "Test Loss: 0.426 | Accuracy: 83.333\n",
      "Validation loss decreased (0.427505 --> 0.425608).  Saving model ...\n",
      "Validation loss decreased (0.425608 --> 0.424828).  Saving model ...\n",
      "Validation loss decreased (0.424828 --> 0.424565).  Saving model ...\n",
      "Validation loss decreased (0.424565 --> 0.423998).  Saving model ...\n",
      "Validation loss decreased (0.423998 --> 0.416129).  Saving model ...\n",
      "Validation loss decreased (0.416129 --> 0.414942).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.414942 --> 0.412761).  Saving model ...\n",
      "Validation loss decreased (0.412761 --> 0.408527).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 90\n",
      "Train Loss: 0.399 | Accuracy: 83.333\n",
      "Test Loss: 0.411 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.408527 --> 0.405068).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.405068 --> 0.400236).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.400236 --> 0.398219).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 100\n",
      "Train Loss: 0.403 | Accuracy: 83.333\n",
      "Test Loss: 0.433 | Accuracy: 83.333\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.398219 --> 0.397727).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.397727 --> 0.396898).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.396898 --> 0.395380).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.395380 --> 0.392586).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 110\n",
      "Train Loss: 0.382 | Accuracy: 83.333\n",
      "Test Loss: 0.400 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.392586 --> 0.391767).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.391767 --> 0.390988).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 120\n",
      "Train Loss: 0.378 | Accuracy: 83.333\n",
      "Test Loss: 0.394 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.390988 --> 0.389738).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.389738 --> 0.389042).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 130\n",
      "Train Loss: 0.382 | Accuracy: 83.333\n",
      "Test Loss: 0.430 | Accuracy: 83.333\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.389042 --> 0.388940).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.388940 --> 0.388715).  Saving model ...\n",
      "Validation loss decreased (0.388715 --> 0.386023).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 140\n",
      "Train Loss: 0.370 | Accuracy: 83.333\n",
      "Test Loss: 0.385 | Accuracy: 83.333\n",
      "Validation loss decreased (0.386023 --> 0.385359).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.385359 --> 0.384893).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.384893 --> 0.384207).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 150\n",
      "Train Loss: 0.366 | Accuracy: 84.549\n",
      "Test Loss: 0.388 | Accuracy: 84.028\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "Validation loss decreased (0.384207 --> 0.384162).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.384162 --> 0.384155).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 160\n",
      "Train Loss: 0.363 | Accuracy: 85.590\n",
      "Test Loss: 0.395 | Accuracy: 84.028\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.384155 --> 0.383208).  Saving model ...\n",
      "Validation loss decreased (0.383208 --> 0.382795).  Saving model ...\n",
      "Validation loss decreased (0.382795 --> 0.382629).  Saving model ...\n",
      "Validation loss decreased (0.382629 --> 0.381681).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 170\n",
      "Train Loss: 0.364 | Accuracy: 84.201\n",
      "Test Loss: 0.385 | Accuracy: 85.417\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.381681 --> 0.381339).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.381339 --> 0.374000).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 180\n",
      "Train Loss: 0.359 | Accuracy: 85.417\n",
      "Test Loss: 0.376 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Validation loss decreased (0.374000 --> 0.373899).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 190\n",
      "Train Loss: 0.350 | Accuracy: 85.938\n",
      "Test Loss: 0.400 | Accuracy: 82.639\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.373899 --> 0.373052).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Split 4 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.619 | Accuracy: 83.333\n",
      "Test Loss: 0.606 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.606332).  Saving model ...\n",
      "Validation loss decreased (0.606332 --> 0.568365).  Saving model ...\n",
      "Validation loss decreased (0.568365 --> 0.489799).  Saving model ...\n",
      "Validation loss decreased (0.489799 --> 0.450100).  Saving model ...\n",
      "Validation loss decreased (0.450100 --> 0.449610).  Saving model ...\n",
      "Validation loss decreased (0.449610 --> 0.449269).  Saving model ...\n",
      "Validation loss decreased (0.449269 --> 0.449103).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.449103 --> 0.448824).  Saving model ...\n",
      "Validation loss decreased (0.448824 --> 0.448694).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.450 | Accuracy: 83.333\n",
      "Test Loss: 0.449 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.448694 --> 0.448598).  Saving model ...\n",
      "Validation loss decreased (0.448598 --> 0.448443).  Saving model ...\n",
      "Validation loss decreased (0.448443 --> 0.448419).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.448419 --> 0.448064).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.448064 --> 0.447956).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.447 | Accuracy: 83.333\n",
      "Test Loss: 0.448 | Accuracy: 83.333\n",
      "Validation loss decreased (0.447956 --> 0.447690).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.447690 --> 0.447642).  Saving model ...\n",
      "Validation loss decreased (0.447642 --> 0.447423).  Saving model ...\n",
      "Validation loss decreased (0.447423 --> 0.447236).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 30\n",
      "Train Loss: 0.446 | Accuracy: 83.333\n",
      "Test Loss: 0.447 | Accuracy: 83.333\n",
      "Validation loss decreased (0.447236 --> 0.447048).  Saving model ...\n",
      "Validation loss decreased (0.447048 --> 0.446944).  Saving model ...\n",
      "Validation loss decreased (0.446944 --> 0.446788).  Saving model ...\n",
      "Validation loss decreased (0.446788 --> 0.446727).  Saving model ...\n",
      "Validation loss decreased (0.446727 --> 0.446602).  Saving model ...\n",
      "Validation loss decreased (0.446602 --> 0.446557).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.446557 --> 0.446283).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 40\n",
      "Train Loss: 0.447 | Accuracy: 83.333\n",
      "Test Loss: 0.446 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.446283 --> 0.445971).  Saving model ...\n",
      "Validation loss decreased (0.445971 --> 0.445955).  Saving model ...\n",
      "Validation loss decreased (0.445955 --> 0.445648).  Saving model ...\n",
      "Validation loss decreased (0.445648 --> 0.445570).  Saving model ...\n",
      "Validation loss decreased (0.445570 --> 0.445398).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.445398 --> 0.445108).  Saving model ...\n",
      "Validation loss decreased (0.445108 --> 0.444864).  Saving model ...\n",
      "Validation loss decreased (0.444864 --> 0.444666).  Saving model ...\n",
      "Epoch: 50\n",
      "Train Loss: 0.444 | Accuracy: 83.333\n",
      "Test Loss: 0.444 | Accuracy: 83.333\n",
      "Validation loss decreased (0.444666 --> 0.444437).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.444437 --> 0.444077).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.444077 --> 0.443693).  Saving model ...\n",
      "Validation loss decreased (0.443693 --> 0.443317).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.443317 --> 0.443081).  Saving model ...\n",
      "Validation loss decreased (0.443081 --> 0.442551).  Saving model ...\n",
      "Validation loss decreased (0.442551 --> 0.442219).  Saving model ...\n",
      "Validation loss decreased (0.442219 --> 0.441029).  Saving model ...\n",
      "Epoch: 60\n",
      "Train Loss: 0.438 | Accuracy: 83.333\n",
      "Test Loss: 0.440 | Accuracy: 83.333\n",
      "Validation loss decreased (0.441029 --> 0.440475).  Saving model ...\n",
      "Validation loss decreased (0.440475 --> 0.440166).  Saving model ...\n",
      "Validation loss decreased (0.440166 --> 0.439148).  Saving model ...\n",
      "Validation loss decreased (0.439148 --> 0.438231).  Saving model ...\n",
      "Validation loss decreased (0.438231 --> 0.437097).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.437097 --> 0.434500).  Saving model ...\n",
      "Validation loss decreased (0.434500 --> 0.433415).  Saving model ...\n",
      "Validation loss decreased (0.433415 --> 0.430732).  Saving model ...\n",
      "Validation loss decreased (0.430732 --> 0.428624).  Saving model ...\n",
      "Epoch: 70\n",
      "Train Loss: 0.422 | Accuracy: 83.333\n",
      "Test Loss: 0.425 | Accuracy: 83.333\n",
      "Validation loss decreased (0.428624 --> 0.424786).  Saving model ...\n",
      "Validation loss decreased (0.424786 --> 0.420364).  Saving model ...\n",
      "Validation loss decreased (0.420364 --> 0.416732).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.416732 --> 0.412642).  Saving model ...\n",
      "Validation loss decreased (0.412642 --> 0.411163).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.411163 --> 0.405720).  Saving model ...\n",
      "Validation loss decreased (0.405720 --> 0.404957).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 80\n",
      "Train Loss: 0.404 | Accuracy: 83.333\n",
      "Test Loss: 0.406 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.404957 --> 0.401998).  Saving model ...\n",
      "Validation loss decreased (0.401998 --> 0.400609).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.400609 --> 0.398274).  Saving model ...\n",
      "Validation loss decreased (0.398274 --> 0.397753).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.397753 --> 0.397091).  Saving model ...\n",
      "Epoch: 90\n",
      "Train Loss: 0.391 | Accuracy: 83.333\n",
      "Test Loss: 0.397 | Accuracy: 83.333\n",
      "Validation loss decreased (0.397091 --> 0.396653).  Saving model ...\n",
      "Validation loss decreased (0.396653 --> 0.395153).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.395153 --> 0.394282).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.394282 --> 0.393323).  Saving model ...\n",
      "Validation loss decreased (0.393323 --> 0.393209).  Saving model ...\n",
      "Epoch: 100\n",
      "Train Loss: 0.387 | Accuracy: 83.333\n",
      "Test Loss: 0.393 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.393209 --> 0.392199).  Saving model ...\n",
      "Validation loss decreased (0.392199 --> 0.390017).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.390017 --> 0.389094).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 110\n",
      "Train Loss: 0.380 | Accuracy: 83.333\n",
      "Test Loss: 0.394 | Accuracy: 83.333\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.389094 --> 0.388395).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.388395 --> 0.387565).  Saving model ...\n",
      "Validation loss decreased (0.387565 --> 0.387024).  Saving model ...\n",
      "Validation loss decreased (0.387024 --> 0.386833).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 120\n",
      "Train Loss: 0.376 | Accuracy: 83.333\n",
      "Test Loss: 0.385 | Accuracy: 83.333\n",
      "Validation loss decreased (0.386833 --> 0.385479).  Saving model ...\n",
      "Validation loss decreased (0.385479 --> 0.385251).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Epoch: 130\n",
      "Train Loss: 0.371 | Accuracy: 83.333\n",
      "Test Loss: 0.388 | Accuracy: 83.333\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 5 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=50, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.716 | Accuracy: 16.667\n",
      "Test Loss: 0.700 | Accuracy: 16.667\n",
      "Validation loss decreased (inf --> 0.700384).  Saving model ...\n",
      "Validation loss decreased (0.700384 --> 0.634491).  Saving model ...\n",
      "Validation loss decreased (0.634491 --> 0.473166).  Saving model ...\n",
      "Validation loss decreased (0.473166 --> 0.457025).  Saving model ...\n",
      "Validation loss decreased (0.457025 --> 0.449428).  Saving model ...\n",
      "Validation loss decreased (0.449428 --> 0.449186).  Saving model ...\n",
      "Validation loss decreased (0.449186 --> 0.449035).  Saving model ...\n",
      "Validation loss decreased (0.449035 --> 0.448931).  Saving model ...\n",
      "Validation loss decreased (0.448931 --> 0.448886).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.448 | Accuracy: 83.333\n",
      "Test Loss: 0.449 | Accuracy: 83.333\n",
      "Validation loss decreased (0.448886 --> 0.448759).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.448759 --> 0.448655).  Saving model ...\n",
      "Validation loss decreased (0.448655 --> 0.448485).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.448485 --> 0.448403).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.444 | Accuracy: 83.333\n",
      "Test Loss: 0.448 | Accuracy: 83.333\n",
      "Validation loss decreased (0.448403 --> 0.448335).  Saving model ...\n",
      "Validation loss decreased (0.448335 --> 0.448290).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.448290 --> 0.448184).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.448184 --> 0.447983).  Saving model ...\n",
      "Validation loss decreased (0.447983 --> 0.447906).  Saving model ...\n",
      "Validation loss decreased (0.447906 --> 0.447873).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.443 | Accuracy: 83.333\n",
      "Test Loss: 0.448 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.447873 --> 0.447660).  Saving model ...\n",
      "Validation loss decreased (0.447660 --> 0.447445).  Saving model ...\n",
      "Validation loss decreased (0.447445 --> 0.447403).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.447403 --> 0.447184).  Saving model ...\n",
      "Validation loss decreased (0.447184 --> 0.447016).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.447016 --> 0.446815).  Saving model ...\n",
      "Epoch: 40\n",
      "Train Loss: 0.439 | Accuracy: 83.333\n",
      "Test Loss: 0.447 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.446815 --> 0.446271).  Saving model ...\n",
      "Validation loss decreased (0.446271 --> 0.445986).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.445986 --> 0.445568).  Saving model ...\n",
      "Validation loss decreased (0.445568 --> 0.445306).  Saving model ...\n",
      "Validation loss decreased (0.445306 --> 0.444927).  Saving model ...\n",
      "Validation loss decreased (0.444927 --> 0.444565).  Saving model ...\n",
      "Validation loss decreased (0.444565 --> 0.444138).  Saving model ...\n",
      "Validation loss decreased (0.444138 --> 0.443847).  Saving model ...\n",
      "Epoch: 50\n",
      "Train Loss: 0.430 | Accuracy: 83.333\n",
      "Test Loss: 0.444 | Accuracy: 83.333\n",
      "Validation loss decreased (0.443847 --> 0.443616).  Saving model ...\n",
      "Validation loss decreased (0.443616 --> 0.442857).  Saving model ...\n",
      "Validation loss decreased (0.442857 --> 0.442260).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.442260 --> 0.441362).  Saving model ...\n",
      "Validation loss decreased (0.441362 --> 0.441236).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 60\n",
      "Train Loss: 0.415 | Accuracy: 83.333\n",
      "Test Loss: 0.441 | Accuracy: 83.333\n",
      "Validation loss decreased (0.441236 --> 0.441130).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.441130 --> 0.440844).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.440844 --> 0.438550).  Saving model ...\n",
      "Epoch: 70\n",
      "Train Loss: 0.389 | Accuracy: 83.333\n",
      "Test Loss: 0.448 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.438550 --> 0.433658).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.433658 --> 0.431776).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.431776 --> 0.429619).  Saving model ...\n",
      "Validation loss decreased (0.429619 --> 0.421426).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.421426 --> 0.421188).  Saving model ...\n",
      "Epoch: 80\n",
      "Train Loss: 0.370 | Accuracy: 83.333\n",
      "Test Loss: 0.434 | Accuracy: 83.333\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.421188 --> 0.420648).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.420648 --> 0.420622).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 90\n",
      "Train Loss: 0.371 | Accuracy: 83.333\n",
      "Test Loss: 0.423 | Accuracy: 83.333\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "train_losses = []\n",
    "train_acc = []\n",
    "#test_acc = []\n",
    "test_losses= []\n",
    "metrics = []\n",
    "#for each fold\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    print('Split '+str(i+1)+' ......')\n",
    "    train_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/train_split'+str(i+1)+'.csv')\n",
    "    test_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/test_split'+str(i+1)+'.csv')\n",
    "    train_loader = data_process(train_set,32)\n",
    "    test_loader = data_process(test_set,len(test_set))\n",
    "    \n",
    "    model = GAT().float().to(device)\n",
    "    \n",
    "    model.apply(reset_weights)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=0.00075,weight_decay=0.001)\n",
    "    \n",
    "    #for each epoch\n",
    "    train_loss = []\n",
    "    test_loss = []\n",
    "    train_accuracy = []\n",
    "    test_accuracy = []\n",
    "    \n",
    "    early_stopping = EarlyStopping(patience=10, verbose=True)\n",
    "    #update the model parameters with 100 epochs\n",
    "    for epoch in range(200):\n",
    "        train(epoch,train_loader)\n",
    "        test(epoch,test_loader)\n",
    "        \n",
    "        early_stopping(test_loss[-1],model)\n",
    "        \n",
    "        if early_stopping.early_stop:\n",
    "            print('Early stopping')\n",
    "            break\n",
    "    \n",
    "    model.load_state_dict(torch.load('checkpoint.pt'))\n",
    "    auc, report = test_metrics(test_loader)\n",
    "    metric = [auc,report]\n",
    "    \n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    train_acc.append(train_accuracy)\n",
    "    test_losses.append(test_loss)\n",
    "    metrics.append(metric)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.7746527777777776 +/- 0.05371921670898749\n",
      "Accuracy: 0.8347222222222224 +/- 0.002777777777777768\n",
      "Precision: 0.717283950617284 +/- 0.04567901234567904\n",
      "Recall: 0.8347222222222224 +/- 0.002777777777777768\n",
      "F1-score: 0.7677956030897208 +/- 0.02043969102792631\n"
     ]
    }
   ],
   "source": [
    "print_metrics(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MorganFingerprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MFP(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MFP, self).__init__()    \n",
    "        #fully connected layers\n",
    "        self.linear1 = Linear(2048, 100)\n",
    "        self.linear2 = Linear(100, 25)\n",
    "        self.linear3 = Linear(25, 10)\n",
    "        self.linear4 = Linear(10, 1)\n",
    "\n",
    "    def forward(self, data):\n",
    "        \n",
    "        mol_feature = data.mol_feature\n",
    "        x = mol_feature.view(data.num_graphs,2048)\n",
    "        x = F.relu(self.linear1(x.float()))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = F.relu(self.linear3(x))\n",
    "        x = torch.sigmoid(self.linear4(x))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1 ......\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.686 | Accuracy: 86.285\n",
      "Test Loss: 0.683 | Accuracy: 88.889\n",
      "Validation loss decreased (inf --> 0.682557).  Saving model ...\n",
      "Validation loss decreased (0.682557 --> 0.670037).  Saving model ...\n",
      "Validation loss decreased (0.670037 --> 0.648369).  Saving model ...\n",
      "Validation loss decreased (0.648369 --> 0.637158).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.637158 --> 0.634424).  Saving model ...\n",
      "Validation loss decreased (0.634424 --> 0.625924).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.538 | Accuracy: 94.792\n",
      "Test Loss: 0.631 | Accuracy: 90.278\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.625924 --> 0.573788).  Saving model ...\n",
      "Validation loss decreased (0.573788 --> 0.334523).  Saving model ...\n",
      "Validation loss decreased (0.334523 --> 0.333840).  Saving model ...\n",
      "Validation loss decreased (0.333840 --> 0.323839).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.323839 --> 0.277693).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.045 | Accuracy: 98.090\n",
      "Test Loss: 0.287 | Accuracy: 89.583\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 2 ......\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.658 | Accuracy: 61.632\n",
      "Test Loss: 0.532 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.531686).  Saving model ...\n",
      "Validation loss decreased (0.531686 --> 0.356334).  Saving model ...\n",
      "Validation loss decreased (0.356334 --> 0.291566).  Saving model ...\n",
      "Validation loss decreased (0.291566 --> 0.278643).  Saving model ...\n",
      "Validation loss decreased (0.278643 --> 0.262590).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.019 | Accuracy: 100.000\n",
      "Test Loss: 0.350 | Accuracy: 93.056\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 3 ......\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.647 | Accuracy: 83.333\n",
      "Test Loss: 0.595 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.595387).  Saving model ...\n",
      "Validation loss decreased (0.595387 --> 0.437934).  Saving model ...\n",
      "Validation loss decreased (0.437934 --> 0.370550).  Saving model ...\n",
      "Validation loss decreased (0.370550 --> 0.334543).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.042 | Accuracy: 99.132\n",
      "Test Loss: 0.336 | Accuracy: 93.056\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 4 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.610 | Accuracy: 83.333\n",
      "Test Loss: 0.555 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.555117).  Saving model ...\n",
      "Validation loss decreased (0.555117 --> 0.411761).  Saving model ...\n",
      "Validation loss decreased (0.411761 --> 0.336788).  Saving model ...\n",
      "Validation loss decreased (0.336788 --> 0.306883).  Saving model ...\n",
      "Validation loss decreased (0.306883 --> 0.277751).  Saving model ...\n",
      "Validation loss decreased (0.277751 --> 0.254807).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.017 | Accuracy: 99.653\n",
      "Test Loss: 0.324 | Accuracy: 90.278\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 5 ......\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.733 | Accuracy: 21.701\n",
      "Test Loss: 0.679 | Accuracy: 63.194\n",
      "Validation loss decreased (inf --> 0.678849).  Saving model ...\n",
      "Validation loss decreased (0.678849 --> 0.464681).  Saving model ...\n",
      "Validation loss decreased (0.464681 --> 0.383596).  Saving model ...\n",
      "Validation loss decreased (0.383596 --> 0.327315).  Saving model ...\n",
      "Validation loss decreased (0.327315 --> 0.289763).  Saving model ...\n",
      "Validation loss decreased (0.289763 --> 0.271337).  Saving model ...\n",
      "Validation loss decreased (0.271337 --> 0.265960).  Saving model ...\n",
      "Validation loss decreased (0.265960 --> 0.263315).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.017 | Accuracy: 99.826\n",
      "Test Loss: 0.304 | Accuracy: 91.667\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "train_losses = []\n",
    "train_acc = []\n",
    "#test_acc = []\n",
    "test_losses= []\n",
    "metrics = []\n",
    "#for each fold\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    print('Split '+str(i+1)+' ......')\n",
    "    train_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/train_split'+str(i+1)+'.csv')\n",
    "    test_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/test_split'+str(i+1)+'.csv')\n",
    "    train_loader = data_process(train_set,32)\n",
    "    test_loader = data_process(test_set,len(test_set))\n",
    "    \n",
    "    model = MFP().float().to(device)\n",
    "    \n",
    "    model.apply(reset_weights)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=0.001,weight_decay=0.001)\n",
    "    \n",
    "    #for each epoch\n",
    "    train_loss = []\n",
    "    test_loss = []\n",
    "    train_accuracy = []\n",
    "    test_accuracy = []\n",
    "    \n",
    "    early_stopping = EarlyStopping(patience=10, verbose=True)\n",
    "    #update the model parameters with 100 epochs\n",
    "    for epoch in range(100):\n",
    "        train(epoch,train_loader)\n",
    "        test(epoch,test_loader)\n",
    "        \n",
    "        early_stopping(test_loss[-1],model)\n",
    "        \n",
    "        if early_stopping.early_stop:\n",
    "            print('Early stopping')\n",
    "            break\n",
    "    \n",
    "    model.load_state_dict(torch.load('checkpoint.pt'))\n",
    "    auc, report = test_metrics(test_loader)\n",
    "    metric = [auc,report]\n",
    "    \n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    train_acc.append(train_accuracy)\n",
    "    test_losses.append(test_loss)\n",
    "    metrics.append(metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.8888194444444444 +/- 0.02854360969325686\n",
      "Accuracy: 0.8972222222222224 +/- 0.03497132864448381\n",
      "Precision: 0.8665271456368844 +/- 0.08769662743191106\n",
      "Recall: 0.8972222222222224 +/- 0.03497132864448381\n",
      "F1-score: 0.877636318589845 +/- 0.06175434808390926\n"
     ]
    }
   ],
   "source": [
    "print_metrics(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transformer(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Transformer, self).__init__()\n",
    "        \n",
    "        #vocab_len to decide how many unique 'word', 1024 the dimension of each embeddings\n",
    "        self.embedding = Embedding.from_pretrained(torch.from_numpy(wv.vectors),freeze=False)\n",
    "        \n",
    "        encoder_layer = TransformerEncoderLayer(d_model=1024, nhead=4)\n",
    "        \n",
    "        self.encoder = TransformerEncoder(encoder_layer,2)\n",
    "        self.pos_encoder = PositionalEncoding(1024, 0.1)\n",
    "        \n",
    "        #fully connected layers\n",
    "        self.linear1 = Linear(1024, 100)\n",
    "        self.linear2 = Linear(100, 25)\n",
    "        self.linear3 = Linear(25, 10)\n",
    "        self.linear4 = Linear(10, 1)\n",
    "\n",
    "    def forward(self, data):\n",
    "        #list of variable length lists of indices\n",
    "        wv_index = data.emb\n",
    "        wv_index = [torch.tensor(item) for item in data.emb]\n",
    "        \n",
    "        padded = pad_sequence(wv_index,padding_value = 266)\n",
    "        # seq_len x batch_size \n",
    "        \n",
    "        fp = self.embedding(padded.to(device))\n",
    "        #seq_len x batch_size x dim_embedding\n",
    "        \n",
    "        fp = self.pos_encoder(fp)\n",
    "        fp = self.encoder(fp)\n",
    "        #seq_len x batch_size x dim_embedding\n",
    "        #learned smile representation \n",
    "        \n",
    "        x = fp.sum(dim=0)\n",
    "        # batch_size x 512 \n",
    "        \n",
    "        # now train a fully connected graph classification network  \n",
    "        x = F.relu(self.linear1(x.float()))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = F.relu(self.linear3(x))\n",
    "        x = torch.sigmoid(self.linear4(x))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.592 | Accuracy: 83.333\n",
      "Test Loss: 0.482 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.481527).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.481527 --> 0.463570).  Saving model ...\n",
      "Validation loss decreased (0.463570 --> 0.449858).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.449858 --> 0.380446).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.380446 --> 0.362069).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.341 | Accuracy: 85.938\n",
      "Test Loss: 0.339 | Accuracy: 88.889\n",
      "Validation loss decreased (0.362069 --> 0.338646).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.222 | Accuracy: 91.319\n",
      "Test Loss: 0.375 | Accuracy: 84.722\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 2 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.810 | Accuracy: 80.729\n",
      "Test Loss: 0.672 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.672249).  Saving model ...\n",
      "Validation loss decreased (0.672249 --> 0.391076).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "Validation loss decreased (0.391076 --> 0.382220).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.320 | Accuracy: 83.333\n",
      "Test Loss: 0.377 | Accuracy: 83.333\n",
      "Validation loss decreased (0.382220 --> 0.376871).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.376871 --> 0.360629).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.256 | Accuracy: 83.333\n",
      "Test Loss: 0.461 | Accuracy: 83.333\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 3 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 1.171 | Accuracy: 76.042\n",
      "Test Loss: 0.495 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.495190).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.495190 --> 0.433861).  Saving model ...\n",
      "Validation loss decreased (0.433861 --> 0.421256).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.421256 --> 0.380039).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.398 | Accuracy: 85.590\n",
      "Test Loss: 0.373 | Accuracy: 86.806\n",
      "Validation loss decreased (0.380039 --> 0.373097).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.373097 --> 0.342060).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.342060 --> 0.336122).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.249 | Accuracy: 91.146\n",
      "Test Loss: 0.349 | Accuracy: 88.194\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 4 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 1.200 | Accuracy: 77.257\n",
      "Test Loss: 1.200 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 1.200187).  Saving model ...\n",
      "Validation loss decreased (1.200187 --> 0.980851).  Saving model ...\n",
      "Validation loss decreased (0.980851 --> 0.764083).  Saving model ...\n",
      "Validation loss decreased (0.764083 --> 0.451585).  Saving model ...\n",
      "Validation loss decreased (0.451585 --> 0.425689).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.425689 --> 0.376665).  Saving model ...\n",
      "Validation loss decreased (0.376665 --> 0.328375).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.353 | Accuracy: 87.326\n",
      "Test Loss: 0.381 | Accuracy: 85.417\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Validation loss decreased (0.328375 --> 0.318540).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.280 | Accuracy: 88.715\n",
      "Test Loss: 0.474 | Accuracy: 83.333\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 5 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 1.250 | Accuracy: 74.653\n",
      "Test Loss: 1.332 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 1.331568).  Saving model ...\n",
      "Validation loss decreased (1.331568 --> 0.413464).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.413464 --> 0.379317).  Saving model ...\n",
      "Validation loss decreased (0.379317 --> 0.370392).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.370392 --> 0.324838).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.290 | Accuracy: 88.194\n",
      "Test Loss: 0.342 | Accuracy: 87.500\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.324838 --> 0.315684).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.315684 --> 0.301249).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.229 | Accuracy: 89.583\n",
      "Test Loss: 0.369 | Accuracy: 82.639\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.301249 --> 0.279616).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Epoch: 30\n",
      "Train Loss: 0.147 | Accuracy: 94.444\n",
      "Test Loss: 0.359 | Accuracy: 83.333\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "train_losses = []\n",
    "train_acc = []\n",
    "#test_acc = []\n",
    "test_losses= []\n",
    "metrics = []\n",
    "#for each fold\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    print('Split '+str(i+1)+' ......')\n",
    "    train_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/train_split'+str(i+1)+'.csv')\n",
    "    test_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/test_split'+str(i+1)+'.csv')\n",
    "    train_loader = data_process(train_set,32)\n",
    "    test_loader = data_process(test_set,len(test_set))\n",
    "    \n",
    "    model = Transformer().float().to(device)\n",
    "    \n",
    "    model.apply(reset_weights)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=0.000075,weight_decay=0.001)\n",
    "    \n",
    "    #for each epoch\n",
    "    train_loss = []\n",
    "    test_loss = []\n",
    "    train_accuracy = []\n",
    "    test_accuracy = []\n",
    "    \n",
    "    early_stopping = EarlyStopping(patience=10, verbose=True)\n",
    "    #update the model parameters with 100 epochs\n",
    "    for epoch in range(100):\n",
    "        train(epoch,train_loader)\n",
    "        test(epoch,test_loader)\n",
    "        \n",
    "        early_stopping(test_loss[-1],model)\n",
    "        \n",
    "        if early_stopping.early_stop:\n",
    "            print('Early stopping')\n",
    "            break\n",
    "    \n",
    "    model.load_state_dict(torch.load('checkpoint.pt'))\n",
    "    auc, report = test_metrics(test_loader)\n",
    "    metric = [auc,report]\n",
    "    \n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    train_acc.append(train_accuracy)\n",
    "    test_losses.append(test_loss)\n",
    "    metrics.append(metric)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.8544791666666667 +/- 0.026545224047897695\n",
      "Accuracy: 0.8805555555555555 +/- 0.027568657279554435\n",
      "Precision: 0.8483710769185524 +/- 0.07920078052845107\n",
      "Recall: 0.8805555555555555 +/- 0.027568657279554435\n",
      "F1-score: 0.8601858397143568 +/- 0.05395394315015024\n"
     ]
    }
   ],
   "source": [
    "print_metrics(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combination with 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transformer + GAT\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GAT_Transformer(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(GAT_Transformer, self).__init__()\n",
    "    \n",
    "        #before the Attention Mechanism, parse in a fully connected network and output a 50 dimensional vector \n",
    "        self.hidden = 50  \n",
    "        self.in_head = 5 #repeat the mechanism for 5 times\n",
    "        \n",
    "        self.conv1 = GATConv(in_channels = 133, \n",
    "                             out_channels = self.hidden,\n",
    "                             heads=self.in_head, concat=False) #set concat to be False, so it will take average instead\n",
    "        \n",
    "        \n",
    "        #define a transformer encoder layer to learn about smiles string\n",
    "        #vocab_len to decide how many unique 'word', 1024 the dimension of each embeddings\n",
    "        self.embedding = Embedding.from_pretrained(torch.from_numpy(wv.vectors),freeze=False)\n",
    "        \n",
    "        encoder_layer = TransformerEncoderLayer(d_model=1024, nhead=2)\n",
    "        \n",
    "        self.encoder = TransformerEncoder(encoder_layer,4)\n",
    "        self.pos_encoder = PositionalEncoding(1024, 0.1)\n",
    "        \n",
    "        #fully connected layers\n",
    "        \n",
    "        self.linear1 = Linear(self.hidden+1024, 100)\n",
    "        self.linear2 = Linear(100, 25)\n",
    "        self.linear3 = Linear(25, 10)\n",
    "        self.linear4 = Linear(10, 1)\n",
    "\n",
    "    def forward(self, data):\n",
    "        \n",
    "        x, edge_index, batch_index, wv_index = data.x, data.edge_index, data.batch, data.emb        \n",
    "        #extract batched node and edge feature matrices, edge_index matrix, batch_index and token indices tensor \n",
    "        \n",
    "        wv_index = [torch.tensor(item) for item in wv_index]\n",
    "        \n",
    "        padded = pad_sequence(wv_index,padding_value = 266)\n",
    "        # seq_len x batch_size \n",
    "        \n",
    "        fp = self.embedding(padded.to(device))\n",
    "        #seq_len x batch_size x dim_embedding\n",
    "        \n",
    "        fp = self.pos_encoder(fp)\n",
    "        fp = self.encoder(fp)\n",
    "        #seq_len x batch_size x dim_embedding\n",
    "        #learned smile representation \n",
    "        \n",
    "        fp = fp.sum(dim=0)\n",
    "        # batch_size x dim_embedding\n",
    "        \n",
    "        x = self.conv1(x.float(), edge_index)\n",
    "\n",
    "        #aggregate the learned local node feature to capture the global property \n",
    "        #here, we apply global_mean_pool over the updated node feature\n",
    "        x = gap(x,batch_index)\n",
    "        \n",
    "        #concatenate x and fp\n",
    "        x = torch.cat([x, fp],dim=1)\n",
    "\n",
    "        # now train a fully connected graph classification network  \n",
    "        x = F.relu(self.linear1(x.float()))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = F.relu(self.linear3(x))\n",
    "        x = torch.sigmoid(self.linear4(x))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1074, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 1.987 | Accuracy: 75.174\n",
      "Test Loss: 0.508 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.507868).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.507868 --> 0.432648).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.432648 --> 0.420889).  Saving model ...\n",
      "Validation loss decreased (0.420889 --> 0.385935).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.417 | Accuracy: 85.069\n",
      "Test Loss: 0.380 | Accuracy: 86.111\n",
      "Validation loss decreased (0.385935 --> 0.379568).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Validation loss decreased (0.379568 --> 0.355545).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.260 | Accuracy: 91.146\n",
      "Test Loss: 0.464 | Accuracy: 84.722\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 2 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1074, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.623 | Accuracy: 83.333\n",
      "Test Loss: 0.695 | Accuracy: 82.639\n",
      "Validation loss decreased (inf --> 0.695297).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.695297 --> 0.647092).  Saving model ...\n",
      "Validation loss decreased (0.647092 --> 0.504832).  Saving model ...\n",
      "Validation loss decreased (0.504832 --> 0.416107).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.494 | Accuracy: 79.514\n",
      "Test Loss: 0.541 | Accuracy: 54.167\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.416107 --> 0.397041).  Saving model ...\n",
      "Validation loss decreased (0.397041 --> 0.374977).  Saving model ...\n",
      "Validation loss decreased (0.374977 --> 0.353389).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.353389 --> 0.348005).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.282 | Accuracy: 88.542\n",
      "Test Loss: 0.397 | Accuracy: 71.528\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 3 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1074, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n",
      "Train Loss: 0.705 | Accuracy: 78.993\n",
      "Test Loss: 0.623 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.622803).  Saving model ...\n",
      "Validation loss decreased (0.622803 --> 0.622479).  Saving model ...\n",
      "Validation loss decreased (0.622479 --> 0.622114).  Saving model ...\n",
      "Validation loss decreased (0.622114 --> 0.621754).  Saving model ...\n",
      "Validation loss decreased (0.621754 --> 0.621393).  Saving model ...\n",
      "Validation loss decreased (0.621393 --> 0.621027).  Saving model ...\n",
      "Validation loss decreased (0.621027 --> 0.620661).  Saving model ...\n",
      "Validation loss decreased (0.620661 --> 0.620297).  Saving model ...\n",
      "Validation loss decreased (0.620297 --> 0.619939).  Saving model ...\n",
      "Validation loss decreased (0.619939 --> 0.619591).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.619 | Accuracy: 83.333\n",
      "Test Loss: 0.619 | Accuracy: 83.333\n",
      "Validation loss decreased (0.619591 --> 0.619223).  Saving model ...\n",
      "Validation loss decreased (0.619223 --> 0.618861).  Saving model ...\n",
      "Validation loss decreased (0.618861 --> 0.618508).  Saving model ...\n",
      "Validation loss decreased (0.618508 --> 0.618157).  Saving model ...\n",
      "Validation loss decreased (0.618157 --> 0.617797).  Saving model ...\n",
      "Validation loss decreased (0.617797 --> 0.617442).  Saving model ...\n",
      "Validation loss decreased (0.617442 --> 0.617078).  Saving model ...\n",
      "Validation loss decreased (0.617078 --> 0.616737).  Saving model ...\n",
      "Validation loss decreased (0.616737 --> 0.616378).  Saving model ...\n",
      "Validation loss decreased (0.616378 --> 0.616033).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.616 | Accuracy: 83.333\n",
      "Test Loss: 0.616 | Accuracy: 83.333\n",
      "Validation loss decreased (0.616033 --> 0.615680).  Saving model ...\n",
      "Validation loss decreased (0.615680 --> 0.615326).  Saving model ...\n",
      "Validation loss decreased (0.615326 --> 0.614976).  Saving model ...\n",
      "Validation loss decreased (0.614976 --> 0.614632).  Saving model ...\n",
      "Validation loss decreased (0.614632 --> 0.614282).  Saving model ...\n",
      "Validation loss decreased (0.614282 --> 0.613937).  Saving model ...\n",
      "Validation loss decreased (0.613937 --> 0.613598).  Saving model ...\n",
      "Validation loss decreased (0.613598 --> 0.613244).  Saving model ...\n",
      "Validation loss decreased (0.613244 --> 0.612904).  Saving model ...\n",
      "Validation loss decreased (0.612904 --> 0.612550).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.612 | Accuracy: 83.333\n",
      "Test Loss: 0.612 | Accuracy: 83.333\n",
      "Validation loss decreased (0.612550 --> 0.612202).  Saving model ...\n",
      "Validation loss decreased (0.612202 --> 0.611866).  Saving model ...\n",
      "Validation loss decreased (0.611866 --> 0.611517).  Saving model ...\n",
      "Validation loss decreased (0.611517 --> 0.611180).  Saving model ...\n",
      "Validation loss decreased (0.611180 --> 0.610832).  Saving model ...\n",
      "Validation loss decreased (0.610832 --> 0.610493).  Saving model ...\n",
      "Validation loss decreased (0.610493 --> 0.610143).  Saving model ...\n",
      "Validation loss decreased (0.610143 --> 0.609815).  Saving model ...\n",
      "Validation loss decreased (0.609815 --> 0.609475).  Saving model ...\n",
      "Validation loss decreased (0.609475 --> 0.609134).  Saving model ...\n",
      "Epoch: 40\n",
      "Train Loss: 0.609 | Accuracy: 83.333\n",
      "Test Loss: 0.609 | Accuracy: 83.333\n",
      "Validation loss decreased (0.609134 --> 0.608793).  Saving model ...\n",
      "Validation loss decreased (0.608793 --> 0.608461).  Saving model ...\n",
      "Validation loss decreased (0.608461 --> 0.608118).  Saving model ...\n",
      "Validation loss decreased (0.608118 --> 0.607773).  Saving model ...\n",
      "Validation loss decreased (0.607773 --> 0.607442).  Saving model ...\n",
      "Validation loss decreased (0.607442 --> 0.607114).  Saving model ...\n",
      "Validation loss decreased (0.607114 --> 0.606779).  Saving model ...\n",
      "Validation loss decreased (0.606779 --> 0.606440).  Saving model ...\n",
      "Validation loss decreased (0.606440 --> 0.606104).  Saving model ...\n",
      "Validation loss decreased (0.606104 --> 0.605770).  Saving model ...\n",
      "Epoch: 50\n",
      "Train Loss: 0.606 | Accuracy: 83.333\n",
      "Test Loss: 0.605 | Accuracy: 83.333\n",
      "Validation loss decreased (0.605770 --> 0.605451).  Saving model ...\n",
      "Validation loss decreased (0.605451 --> 0.605111).  Saving model ...\n",
      "Validation loss decreased (0.605111 --> 0.604773).  Saving model ...\n",
      "Validation loss decreased (0.604773 --> 0.604442).  Saving model ...\n",
      "Validation loss decreased (0.604442 --> 0.604117).  Saving model ...\n",
      "Validation loss decreased (0.604117 --> 0.603787).  Saving model ...\n",
      "Validation loss decreased (0.603787 --> 0.603453).  Saving model ...\n",
      "Validation loss decreased (0.603453 --> 0.603126).  Saving model ...\n",
      "Validation loss decreased (0.603126 --> 0.602796).  Saving model ...\n",
      "Validation loss decreased (0.602796 --> 0.602470).  Saving model ...\n",
      "Epoch: 60\n",
      "Train Loss: 0.602 | Accuracy: 83.333\n",
      "Test Loss: 0.602 | Accuracy: 83.333\n",
      "Validation loss decreased (0.602470 --> 0.602138).  Saving model ...\n",
      "Validation loss decreased (0.602138 --> 0.601806).  Saving model ...\n",
      "Validation loss decreased (0.601806 --> 0.601484).  Saving model ...\n",
      "Validation loss decreased (0.601484 --> 0.601166).  Saving model ...\n",
      "Validation loss decreased (0.601166 --> 0.600836).  Saving model ...\n",
      "Validation loss decreased (0.600836 --> 0.600507).  Saving model ...\n",
      "Validation loss decreased (0.600507 --> 0.600183).  Saving model ...\n",
      "Validation loss decreased (0.600183 --> 0.599863).  Saving model ...\n",
      "Validation loss decreased (0.599863 --> 0.599543).  Saving model ...\n",
      "Validation loss decreased (0.599543 --> 0.599212).  Saving model ...\n",
      "Epoch: 70\n",
      "Train Loss: 0.599 | Accuracy: 83.333\n",
      "Test Loss: 0.599 | Accuracy: 83.333\n",
      "Validation loss decreased (0.599212 --> 0.598701).  Saving model ...\n",
      "Validation loss decreased (0.598701 --> 0.598193).  Saving model ...\n",
      "Validation loss decreased (0.598193 --> 0.597704).  Saving model ...\n",
      "Validation loss decreased (0.597704 --> 0.597224).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.597224 --> 0.597074).  Saving model ...\n",
      "Validation loss decreased (0.597074 --> 0.596764).  Saving model ...\n",
      "Validation loss decreased (0.596764 --> 0.596437).  Saving model ...\n",
      "Validation loss decreased (0.596437 --> 0.596118).  Saving model ...\n",
      "Epoch: 80\n",
      "Train Loss: 0.596 | Accuracy: 83.333\n",
      "Test Loss: 0.596 | Accuracy: 83.333\n",
      "Validation loss decreased (0.596118 --> 0.595802).  Saving model ...\n",
      "Validation loss decreased (0.595802 --> 0.595489).  Saving model ...\n",
      "Validation loss decreased (0.595489 --> 0.595174).  Saving model ...\n",
      "Validation loss decreased (0.595174 --> 0.594855).  Saving model ...\n",
      "Validation loss decreased (0.594855 --> 0.594548).  Saving model ...\n",
      "Validation loss decreased (0.594548 --> 0.594221).  Saving model ...\n",
      "Validation loss decreased (0.594221 --> 0.593915).  Saving model ...\n",
      "Validation loss decreased (0.593915 --> 0.593609).  Saving model ...\n",
      "Validation loss decreased (0.593609 --> 0.593282).  Saving model ...\n",
      "Validation loss decreased (0.593282 --> 0.592966).  Saving model ...\n",
      "Epoch: 90\n",
      "Train Loss: 0.593 | Accuracy: 83.333\n",
      "Test Loss: 0.593 | Accuracy: 83.333\n",
      "Validation loss decreased (0.592966 --> 0.592660).  Saving model ...\n",
      "Validation loss decreased (0.592660 --> 0.592355).  Saving model ...\n",
      "Validation loss decreased (0.592355 --> 0.592042).  Saving model ...\n",
      "Validation loss decreased (0.592042 --> 0.591726).  Saving model ...\n",
      "Validation loss decreased (0.591726 --> 0.591413).  Saving model ...\n",
      "Validation loss decreased (0.591413 --> 0.591108).  Saving model ...\n",
      "Validation loss decreased (0.591108 --> 0.590803).  Saving model ...\n",
      "Validation loss decreased (0.590803 --> 0.590486).  Saving model ...\n",
      "Validation loss decreased (0.590486 --> 0.590181).  Saving model ...\n",
      "Validation loss decreased (0.590181 --> 0.589878).  Saving model ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\jimmy\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 4 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1074, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 1.419 | Accuracy: 76.736\n",
      "Test Loss: 0.586 | Accuracy: 73.611\n",
      "Validation loss decreased (inf --> 0.585739).  Saving model ...\n",
      "Validation loss decreased (0.585739 --> 0.454610).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.454610 --> 0.415285).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.410 | Accuracy: 83.854\n",
      "Test Loss: 0.398 | Accuracy: 84.028\n",
      "Validation loss decreased (0.415285 --> 0.398476).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.398476 --> 0.391914).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.391914 --> 0.372475).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.327 | Accuracy: 87.847\n",
      "Test Loss: 0.514 | Accuracy: 86.806\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 5 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=1074, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.619 | Accuracy: 83.160\n",
      "Test Loss: 0.453 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.452657).  Saving model ...\n",
      "Validation loss decreased (0.452657 --> 0.446721).  Saving model ...\n",
      "Validation loss decreased (0.446721 --> 0.442642).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.442642 --> 0.433931).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.433931 --> 0.417067).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.417067 --> 0.395105).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.390 | Accuracy: 82.465\n",
      "Test Loss: 0.348 | Accuracy: 84.722\n",
      "Validation loss decreased (0.395105 --> 0.347536).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.347536 --> 0.343698).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.343698 --> 0.338668).  Saving model ...\n",
      "Validation loss decreased (0.338668 --> 0.321511).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.315 | Accuracy: 86.979\n",
      "Test Loss: 0.409 | Accuracy: 83.333\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Validation loss decreased (0.321511 --> 0.315931).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30\n",
      "Train Loss: 0.269 | Accuracy: 86.632\n",
      "Test Loss: 0.370 | Accuracy: 84.722\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "train_losses = []\n",
    "train_acc = []\n",
    "#test_acc = []\n",
    "test_losses= []\n",
    "metrics = []\n",
    "#for each fold\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    print('Split '+str(i+1)+' ......')\n",
    "    train_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/train_split'+str(i+1)+'.csv')\n",
    "    test_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/test_split'+str(i+1)+'.csv')\n",
    "    train_loader = data_process(train_set,32)\n",
    "    test_loader = data_process(test_set,len(test_set))\n",
    "    \n",
    "    model = GAT_Transformer().float().to(device)\n",
    "    \n",
    "    model.apply(reset_weights)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=0.000075,weight_decay=0.001)\n",
    "    \n",
    "    #for each epoch\n",
    "    train_loss = []\n",
    "    test_loss = []\n",
    "    train_accuracy = []\n",
    "    test_accuracy = []\n",
    "    \n",
    "    early_stopping = EarlyStopping(patience=10, verbose=True)\n",
    "    #update the model parameters with 100 epochs\n",
    "    for epoch in range(100):\n",
    "        train(epoch,train_loader)\n",
    "        test(epoch,test_loader)\n",
    "        \n",
    "        early_stopping(test_loss[-1],model)\n",
    "        \n",
    "        if early_stopping.early_stop:\n",
    "            print('Early stopping')\n",
    "            break\n",
    "    \n",
    "    model.load_state_dict(torch.load('checkpoint.pt'))\n",
    "    auc, report = test_metrics(test_loader)\n",
    "    metric = [auc,report]\n",
    "    \n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    train_acc.append(train_accuracy)\n",
    "    test_losses.append(test_loss)\n",
    "    metrics.append(metric)\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.7736805555555556 +/- 0.1417788567204679\n",
      "Accuracy: 0.8597222222222222 +/- 0.025382870670134804\n",
      "Precision: 0.8383203605501371 +/- 0.07237372594670383\n",
      "Recall: 0.8597222222222222 +/- 0.025382870670134804\n",
      "F1-score: 0.8430819951807391 +/- 0.044820889338285944\n"
     ]
    }
   ],
   "source": [
    "print_metrics(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transformer + MorganFingerprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transformer_MFP(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Transformer_MFP, self).__init__()\n",
    "    \n",
    "        #before the Attention Mechanism, parse in a fully connected network and output a 50 dimensional vector \n",
    "        \n",
    "        \n",
    "        #define a transformer encoder layer to learn about smiles string\n",
    "        #vocab_len to decide how many unique 'word', 1024 the dimension of each embeddings\n",
    "        self.embedding = Embedding.from_pretrained(torch.from_numpy(wv.vectors),freeze=False)\n",
    "        \n",
    "        encoder_layer = TransformerEncoderLayer(d_model=1024, nhead=2)\n",
    "        \n",
    "        self.encoder = TransformerEncoder(encoder_layer,4)\n",
    "        self.pos_encoder = PositionalEncoding(1024, 0.1)\n",
    "        \n",
    "        #fully connected layers\n",
    "        \n",
    "        self.linear1 = Linear(2048+1024, 100)\n",
    "        self.linear2 = Linear(100, 25)\n",
    "        self.linear3 = Linear(25, 10)\n",
    "        self.linear4 = Linear(10, 1)\n",
    "\n",
    "    def forward(self, data):\n",
    "        \n",
    "        batch_index, wv_index, mol_feature = data.batch, data.emb, data.mol_feature        \n",
    "        #extract batched node and edge feature matrices, edge_index matrix, batch_index and token indices tensor \n",
    "        \n",
    "        wv_index = [torch.tensor(item) for item in wv_index]\n",
    "        \n",
    "        padded = pad_sequence(wv_index,padding_value = 266)\n",
    "        # seq_len x batch_size \n",
    "        \n",
    "        fp = self.embedding(padded.to(device))\n",
    "        #seq_len x batch_size x dim_embedding\n",
    "        \n",
    "        fp = self.pos_encoder(fp)\n",
    "        fp = self.encoder(fp)\n",
    "        #seq_len x batch_size x dim_embedding\n",
    "        #learned smile representation \n",
    "        \n",
    "        fp = fp.sum(dim=0)\n",
    "        # batch_size x dim_embedding\n",
    "        \n",
    "        mol_feature = mol_feature.view(data.num_graphs,2048)\n",
    "        #concatenate x and fp\n",
    "        \n",
    "        x = torch.cat([mol_feature, fp],dim=1)\n",
    "\n",
    "        # now train a fully connected graph classification network  \n",
    "        x = F.relu(self.linear1(x.float()))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = F.relu(self.linear3(x))\n",
    "        x = torch.sigmoid(self.linear4(x))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3072, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.942 | Accuracy: 80.556\n",
      "Test Loss: 0.461 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.461367).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.461367 --> 0.396150).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.396150 --> 0.354820).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.313 | Accuracy: 88.021\n",
      "Test Loss: 0.372 | Accuracy: 90.278\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.354820 --> 0.352416).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.133 | Accuracy: 95.312\n",
      "Test Loss: 0.681 | Accuracy: 89.583\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 2 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3072, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.798 | Accuracy: 77.083\n",
      "Test Loss: 0.601 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.600594).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.600594 --> 0.454445).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.454445 --> 0.416970).  Saving model ...\n",
      "Validation loss decreased (0.416970 --> 0.408392).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.340 | Accuracy: 86.111\n",
      "Test Loss: 0.464 | Accuracy: 84.722\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.408392 --> 0.403654).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.403654 --> 0.339165).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.249 | Accuracy: 91.319\n",
      "Test Loss: 0.788 | Accuracy: 85.417\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 3 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3072, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n",
      "Train Loss: 0.788 | Accuracy: 74.653\n",
      "Test Loss: 0.431 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.430815).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.430815 --> 0.398722).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.362 | Accuracy: 89.583\n",
      "Test Loss: 0.384 | Accuracy: 87.500\n",
      "Validation loss decreased (0.398722 --> 0.383591).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Validation loss decreased (0.383591 --> 0.342554).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.246 | Accuracy: 93.056\n",
      "Test Loss: 0.553 | Accuracy: 86.111\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 4 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3072, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.985 | Accuracy: 78.819\n",
      "Test Loss: 0.615 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.614979).  Saving model ...\n",
      "Validation loss decreased (0.614979 --> 0.512681).  Saving model ...\n",
      "Validation loss decreased (0.512681 --> 0.504649).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.504649 --> 0.481140).  Saving model ...\n",
      "Validation loss decreased (0.481140 --> 0.360257).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.322 | Accuracy: 88.715\n",
      "Test Loss: 0.382 | Accuracy: 88.889\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.360257 --> 0.280131).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.193 | Accuracy: 92.188\n",
      "Test Loss: 0.728 | Accuracy: 87.500\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 5 ......\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3072, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.695 | Accuracy: 76.736\n",
      "Test Loss: 0.484 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.484295).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.484295 --> 0.422408).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.422408 --> 0.399395).  Saving model ...\n",
      "Validation loss decreased (0.399395 --> 0.372034).  Saving model ...\n",
      "Validation loss decreased (0.372034 --> 0.339866).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.377 | Accuracy: 86.632\n",
      "Test Loss: 0.342 | Accuracy: 87.500\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.339866 --> 0.320161).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.320161 --> 0.313186).  Saving model ...\n",
      "Validation loss decreased (0.313186 --> 0.311319).  Saving model ...\n",
      "Validation loss decreased (0.311319 --> 0.295916).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.295916 --> 0.271519).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.221 | Accuracy: 91.319\n",
      "Test Loss: 0.292 | Accuracy: 90.278\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.271519 --> 0.269694).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Epoch: 30\n",
      "Train Loss: 0.179 | Accuracy: 92.708\n",
      "Test Loss: 0.426 | Accuracy: 86.806\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "train_losses = []\n",
    "train_acc = []\n",
    "#test_acc = []\n",
    "test_losses= []\n",
    "metrics = []\n",
    "#for each fold\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    print('Split '+str(i+1)+' ......')\n",
    "    train_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/FYP/train_split'+str(i+1)+'.csv')\n",
    "    test_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/test_split'+str(i+1)+'.csv')\n",
    "    train_loader = data_process(train_set,32)\n",
    "    test_loader = data_process(test_set,len(test_set))\n",
    "    \n",
    "    model = Transformer_MFP().float().to(device)\n",
    "    \n",
    "    model.apply(reset_weights)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=0.000075,weight_decay=0.001)\n",
    "    \n",
    "    #for each epoch\n",
    "    train_loss = []\n",
    "    test_loss = []\n",
    "    train_accuracy = []\n",
    "    test_accuracy = []\n",
    "    \n",
    "    early_stopping = EarlyStopping(patience=10, verbose=True)\n",
    "    #update the model parameters with 100 epochs\n",
    "    for epoch in range(100):\n",
    "        train(epoch,train_loader)\n",
    "        test(epoch,test_loader)\n",
    "        \n",
    "        early_stopping(test_loss[-1],model)\n",
    "        \n",
    "        if early_stopping.early_stop:\n",
    "            print('Early stopping')\n",
    "            break\n",
    "    \n",
    "    model.load_state_dict(torch.load('checkpoint.pt'))\n",
    "    auc, report = test_metrics(test_loader)\n",
    "    metric = [auc,report]\n",
    "    \n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    train_acc.append(train_accuracy)\n",
    "    test_losses.append(test_loss)\n",
    "    metrics.append(metric)\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.8484722222222223 +/- 0.03343200096122882\n",
      "Accuracy: 0.8861111111111111 +/- 0.015590239111558069\n",
      "Precision: 0.8837202520351166 +/- 0.01798267917927629\n",
      "Recall: 0.8861111111111111 +/- 0.015590239111558069\n",
      "F1-score: 0.877027331321184 +/- 0.015559003187079682\n"
     ]
    }
   ],
   "source": [
    "print_metrics(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GAT + MorganFingerprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GAT_MFP(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(GAT_MFP, self).__init__()\n",
    "    \n",
    "        #before the Attention Mechanism, parse in a fully connected network and output a 50 dimensional vector \n",
    "        self.hidden = 50  \n",
    "        self.in_head = 5 #repeat the mechanism for 5 times\n",
    "        self.conv1 = GATConv(in_channels = 133, \n",
    "                             out_channels = self.hidden, \n",
    "                             heads=self.in_head, concat=False) #set concat to be False, so it will take average instead\n",
    "\n",
    "        \n",
    "        #fully connected layers\n",
    "        self.linear1 = Linear(self.hidden+2048,100)\n",
    "        self.linear2 = Linear(100, 25)\n",
    "        self.linear3 = Linear(25, 10)\n",
    "        self.linear4 = Linear(10, 1)\n",
    "\n",
    "    def forward(self, data):\n",
    "        \n",
    "        x, edge_index, batch_index, mol_feature = data.x, data.edge_index, data.batch, data.mol_feature    \n",
    "        #extract node vectors, edge_index, batch index, and binary morgan fingerprint \n",
    "        \n",
    "        x = self.conv1(x.float(), edge_index)\n",
    "\n",
    "        #aggregate the learned local node feature to capture the global property \n",
    "        #here, we apply global_mean_pool over the updated node feature\n",
    "        x = gap(x,batch_index)\n",
    "        #also include some other global information i.e. binary morgan fingerprint\n",
    "        x = torch.cat([x, mol_feature.reshape(data.num_graphs,2048)],dim=1)\n",
    "\n",
    "        x = F.relu(self.linear1(x.float()))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = F.relu(self.linear3(x))\n",
    "        x = torch.sigmoid(self.linear4(x))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=2098, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.600 | Accuracy: 83.333\n",
      "Test Loss: 0.597 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.597306).  Saving model ...\n",
      "Validation loss decreased (0.597306 --> 0.591037).  Saving model ...\n",
      "Validation loss decreased (0.591037 --> 0.582555).  Saving model ...\n",
      "Validation loss decreased (0.582555 --> 0.571094).  Saving model ...\n",
      "Validation loss decreased (0.571094 --> 0.556010).  Saving model ...\n",
      "Validation loss decreased (0.556010 --> 0.537784).  Saving model ...\n",
      "Validation loss decreased (0.537784 --> 0.515588).  Saving model ...\n",
      "Validation loss decreased (0.515588 --> 0.491482).  Saving model ...\n",
      "Validation loss decreased (0.491482 --> 0.466323).  Saving model ...\n",
      "Validation loss decreased (0.466323 --> 0.442741).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.416 | Accuracy: 83.333\n",
      "Test Loss: 0.421 | Accuracy: 83.333\n",
      "Validation loss decreased (0.442741 --> 0.421114).  Saving model ...\n",
      "Validation loss decreased (0.421114 --> 0.403388).  Saving model ...\n",
      "Validation loss decreased (0.403388 --> 0.388127).  Saving model ...\n",
      "Validation loss decreased (0.388127 --> 0.374080).  Saving model ...\n",
      "Validation loss decreased (0.374080 --> 0.362629).  Saving model ...\n",
      "Validation loss decreased (0.362629 --> 0.351865).  Saving model ...\n",
      "Validation loss decreased (0.351865 --> 0.342703).  Saving model ...\n",
      "Validation loss decreased (0.342703 --> 0.334097).  Saving model ...\n",
      "Validation loss decreased (0.334097 --> 0.325464).  Saving model ...\n",
      "Validation loss decreased (0.325464 --> 0.318747).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.274 | Accuracy: 84.896\n",
      "Test Loss: 0.313 | Accuracy: 84.722\n",
      "Validation loss decreased (0.318747 --> 0.313060).  Saving model ...\n",
      "Validation loss decreased (0.313060 --> 0.307408).  Saving model ...\n",
      "Validation loss decreased (0.307408 --> 0.302124).  Saving model ...\n",
      "Validation loss decreased (0.302124 --> 0.295366).  Saving model ...\n",
      "Validation loss decreased (0.295366 --> 0.289195).  Saving model ...\n",
      "Validation loss decreased (0.289195 --> 0.284342).  Saving model ...\n",
      "Validation loss decreased (0.284342 --> 0.278498).  Saving model ...\n",
      "Validation loss decreased (0.278498 --> 0.274552).  Saving model ...\n",
      "Validation loss decreased (0.274552 --> 0.270289).  Saving model ...\n",
      "Validation loss decreased (0.270289 --> 0.265050).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.181 | Accuracy: 93.924\n",
      "Test Loss: 0.262 | Accuracy: 90.278\n",
      "Validation loss decreased (0.265050 --> 0.262304).  Saving model ...\n",
      "Validation loss decreased (0.262304 --> 0.259324).  Saving model ...\n",
      "Validation loss decreased (0.259324 --> 0.254915).  Saving model ...\n",
      "Validation loss decreased (0.254915 --> 0.252182).  Saving model ...\n",
      "Validation loss decreased (0.252182 --> 0.250423).  Saving model ...\n",
      "Validation loss decreased (0.250423 --> 0.246803).  Saving model ...\n",
      "Validation loss decreased (0.246803 --> 0.245702).  Saving model ...\n",
      "Validation loss decreased (0.245702 --> 0.243670).  Saving model ...\n",
      "Validation loss decreased (0.243670 --> 0.241863).  Saving model ...\n",
      "Validation loss decreased (0.241863 --> 0.240839).  Saving model ...\n",
      "Epoch: 40\n",
      "Train Loss: 0.106 | Accuracy: 96.875\n",
      "Test Loss: 0.241 | Accuracy: 90.278\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.240839 --> 0.237687).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Epoch: 50\n",
      "Train Loss: 0.060 | Accuracy: 99.306\n",
      "Test Loss: 0.242 | Accuracy: 90.972\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 2 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=2098, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.659 | Accuracy: 83.333\n",
      "Test Loss: 0.657 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.656589).  Saving model ...\n",
      "Validation loss decreased (0.656589 --> 0.653429).  Saving model ...\n",
      "Validation loss decreased (0.653429 --> 0.649233).  Saving model ...\n",
      "Validation loss decreased (0.649233 --> 0.641354).  Saving model ...\n",
      "Validation loss decreased (0.641354 --> 0.630098).  Saving model ...\n",
      "Validation loss decreased (0.630098 --> 0.615297).  Saving model ...\n",
      "Validation loss decreased (0.615297 --> 0.597251).  Saving model ...\n",
      "Validation loss decreased (0.597251 --> 0.574704).  Saving model ...\n",
      "Validation loss decreased (0.574704 --> 0.549195).  Saving model ...\n",
      "Validation loss decreased (0.549195 --> 0.520899).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.509 | Accuracy: 83.333\n",
      "Test Loss: 0.493 | Accuracy: 83.333\n",
      "Validation loss decreased (0.520899 --> 0.492565).  Saving model ...\n",
      "Validation loss decreased (0.492565 --> 0.463514).  Saving model ...\n",
      "Validation loss decreased (0.463514 --> 0.439444).  Saving model ...\n",
      "Validation loss decreased (0.439444 --> 0.418264).  Saving model ...\n",
      "Validation loss decreased (0.418264 --> 0.400419).  Saving model ...\n",
      "Validation loss decreased (0.400419 --> 0.385022).  Saving model ...\n",
      "Validation loss decreased (0.385022 --> 0.372881).  Saving model ...\n",
      "Validation loss decreased (0.372881 --> 0.362024).  Saving model ...\n",
      "Validation loss decreased (0.362024 --> 0.352481).  Saving model ...\n",
      "Validation loss decreased (0.352481 --> 0.343782).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.321 | Accuracy: 83.333\n",
      "Test Loss: 0.336 | Accuracy: 83.333\n",
      "Validation loss decreased (0.343782 --> 0.335641).  Saving model ...\n",
      "Validation loss decreased (0.335641 --> 0.328951).  Saving model ...\n",
      "Validation loss decreased (0.328951 --> 0.323066).  Saving model ...\n",
      "Validation loss decreased (0.323066 --> 0.317341).  Saving model ...\n",
      "Validation loss decreased (0.317341 --> 0.312280).  Saving model ...\n",
      "Validation loss decreased (0.312280 --> 0.307087).  Saving model ...\n",
      "Validation loss decreased (0.307087 --> 0.302767).  Saving model ...\n",
      "Validation loss decreased (0.302767 --> 0.298169).  Saving model ...\n",
      "Validation loss decreased (0.298169 --> 0.293596).  Saving model ...\n",
      "Validation loss decreased (0.293596 --> 0.289356).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.240 | Accuracy: 92.361\n",
      "Test Loss: 0.286 | Accuracy: 90.278\n",
      "Validation loss decreased (0.289356 --> 0.285937).  Saving model ...\n",
      "Validation loss decreased (0.285937 --> 0.282611).  Saving model ...\n",
      "Validation loss decreased (0.282611 --> 0.279897).  Saving model ...\n",
      "Validation loss decreased (0.279897 --> 0.275807).  Saving model ...\n",
      "Validation loss decreased (0.275807 --> 0.273564).  Saving model ...\n",
      "Validation loss decreased (0.273564 --> 0.271006).  Saving model ...\n",
      "Validation loss decreased (0.271006 --> 0.267332).  Saving model ...\n",
      "Validation loss decreased (0.267332 --> 0.265211).  Saving model ...\n",
      "Validation loss decreased (0.265211 --> 0.261841).  Saving model ...\n",
      "Validation loss decreased (0.261841 --> 0.259494).  Saving model ...\n",
      "Epoch: 40\n",
      "Train Loss: 0.178 | Accuracy: 94.097\n",
      "Test Loss: 0.257 | Accuracy: 93.056\n",
      "Validation loss decreased (0.259494 --> 0.257467).  Saving model ...\n",
      "Validation loss decreased (0.257467 --> 0.253848).  Saving model ...\n",
      "Validation loss decreased (0.253848 --> 0.251646).  Saving model ...\n",
      "Validation loss decreased (0.251646 --> 0.248549).  Saving model ...\n",
      "Validation loss decreased (0.248549 --> 0.246503).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.246503 --> 0.244391).  Saving model ...\n",
      "Validation loss decreased (0.244391 --> 0.241723).  Saving model ...\n",
      "Validation loss decreased (0.241723 --> 0.239103).  Saving model ...\n",
      "Validation loss decreased (0.239103 --> 0.238446).  Saving model ...\n",
      "Validation loss decreased (0.238446 --> 0.235916).  Saving model ...\n",
      "Epoch: 50\n",
      "Train Loss: 0.120 | Accuracy: 96.701\n",
      "Test Loss: 0.234 | Accuracy: 92.361\n",
      "Validation loss decreased (0.235916 --> 0.233836).  Saving model ...\n",
      "Validation loss decreased (0.233836 --> 0.231130).  Saving model ...\n",
      "Validation loss decreased (0.231130 --> 0.230572).  Saving model ...\n",
      "Validation loss decreased (0.230572 --> 0.228926).  Saving model ...\n",
      "Validation loss decreased (0.228926 --> 0.227748).  Saving model ...\n",
      "Validation loss decreased (0.227748 --> 0.226935).  Saving model ...\n",
      "Validation loss decreased (0.226935 --> 0.224399).  Saving model ...\n",
      "Validation loss decreased (0.224399 --> 0.223644).  Saving model ...\n",
      "Validation loss decreased (0.223644 --> 0.222942).  Saving model ...\n",
      "Validation loss decreased (0.222942 --> 0.222607).  Saving model ...\n",
      "Epoch: 60\n",
      "Train Loss: 0.077 | Accuracy: 98.264\n",
      "Test Loss: 0.222 | Accuracy: 92.361\n",
      "Validation loss decreased (0.222607 --> 0.222378).  Saving model ...\n",
      "Validation loss decreased (0.222378 --> 0.221037).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.221037 --> 0.220604).  Saving model ...\n",
      "Validation loss decreased (0.220604 --> 0.219331).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.219331 --> 0.219265).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 70\n",
      "Train Loss: 0.051 | Accuracy: 98.958\n",
      "Test Loss: 0.221 | Accuracy: 92.361\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 3 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=2098, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.626 | Accuracy: 83.333\n",
      "Test Loss: 0.623 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.623118).  Saving model ...\n",
      "Validation loss decreased (0.623118 --> 0.616769).  Saving model ...\n",
      "Validation loss decreased (0.616769 --> 0.610103).  Saving model ...\n",
      "Validation loss decreased (0.610103 --> 0.602203).  Saving model ...\n",
      "Validation loss decreased (0.602203 --> 0.592363).  Saving model ...\n",
      "Validation loss decreased (0.592363 --> 0.579297).  Saving model ...\n",
      "Validation loss decreased (0.579297 --> 0.563054).  Saving model ...\n",
      "Validation loss decreased (0.563054 --> 0.544910).  Saving model ...\n",
      "Validation loss decreased (0.544910 --> 0.525416).  Saving model ...\n",
      "Validation loss decreased (0.525416 --> 0.505163).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.480 | Accuracy: 83.333\n",
      "Test Loss: 0.484 | Accuracy: 83.333\n",
      "Validation loss decreased (0.505163 --> 0.483668).  Saving model ...\n",
      "Validation loss decreased (0.483668 --> 0.462470).  Saving model ...\n",
      "Validation loss decreased (0.462470 --> 0.442990).  Saving model ...\n",
      "Validation loss decreased (0.442990 --> 0.425406).  Saving model ...\n",
      "Validation loss decreased (0.425406 --> 0.409542).  Saving model ...\n",
      "Validation loss decreased (0.409542 --> 0.395769).  Saving model ...\n",
      "Validation loss decreased (0.395769 --> 0.383783).  Saving model ...\n",
      "Validation loss decreased (0.383783 --> 0.373062).  Saving model ...\n",
      "Validation loss decreased (0.373062 --> 0.362111).  Saving model ...\n",
      "Validation loss decreased (0.362111 --> 0.352659).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.295 | Accuracy: 89.931\n",
      "Test Loss: 0.344 | Accuracy: 87.500\n",
      "Validation loss decreased (0.352659 --> 0.343690).  Saving model ...\n",
      "Validation loss decreased (0.343690 --> 0.335855).  Saving model ...\n",
      "Validation loss decreased (0.335855 --> 0.328410).  Saving model ...\n",
      "Validation loss decreased (0.328410 --> 0.321778).  Saving model ...\n",
      "Validation loss decreased (0.321778 --> 0.315260).  Saving model ...\n",
      "Validation loss decreased (0.315260 --> 0.309374).  Saving model ...\n",
      "Validation loss decreased (0.309374 --> 0.305243).  Saving model ...\n",
      "Validation loss decreased (0.305243 --> 0.298966).  Saving model ...\n",
      "Validation loss decreased (0.298966 --> 0.297738).  Saving model ...\n",
      "Validation loss decreased (0.297738 --> 0.290438).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.201 | Accuracy: 93.576\n",
      "Test Loss: 0.288 | Accuracy: 89.583\n",
      "Validation loss decreased (0.290438 --> 0.288430).  Saving model ...\n",
      "Validation loss decreased (0.288430 --> 0.286491).  Saving model ...\n",
      "Validation loss decreased (0.286491 --> 0.281840).  Saving model ...\n",
      "Validation loss decreased (0.281840 --> 0.279426).  Saving model ...\n",
      "Validation loss decreased (0.279426 --> 0.278034).  Saving model ...\n",
      "Validation loss decreased (0.278034 --> 0.273430).  Saving model ...\n",
      "Validation loss decreased (0.273430 --> 0.272675).  Saving model ...\n",
      "Validation loss decreased (0.272675 --> 0.270790).  Saving model ...\n",
      "Validation loss decreased (0.270790 --> 0.266252).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 40\n",
      "Train Loss: 0.137 | Accuracy: 95.312\n",
      "Test Loss: 0.264 | Accuracy: 90.972\n",
      "Validation loss decreased (0.266252 --> 0.263888).  Saving model ...\n",
      "Validation loss decreased (0.263888 --> 0.263769).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.263769 --> 0.261391).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.261391 --> 0.261189).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Epoch: 50\n",
      "Train Loss: 0.088 | Accuracy: 97.049\n",
      "Test Loss: 0.267 | Accuracy: 91.667\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 4 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=2098, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.808 | Accuracy: 16.667\n",
      "Test Loss: 0.802 | Accuracy: 16.667\n",
      "Validation loss decreased (inf --> 0.802361).  Saving model ...\n",
      "Validation loss decreased (0.802361 --> 0.790655).  Saving model ...\n",
      "Validation loss decreased (0.790655 --> 0.774657).  Saving model ...\n",
      "Validation loss decreased (0.774657 --> 0.756078).  Saving model ...\n",
      "Validation loss decreased (0.756078 --> 0.735760).  Saving model ...\n",
      "Validation loss decreased (0.735760 --> 0.710867).  Saving model ...\n",
      "Validation loss decreased (0.710867 --> 0.680476).  Saving model ...\n",
      "Validation loss decreased (0.680476 --> 0.644110).  Saving model ...\n",
      "Validation loss decreased (0.644110 --> 0.603764).  Saving model ...\n",
      "Validation loss decreased (0.603764 --> 0.561772).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.533 | Accuracy: 83.333\n",
      "Test Loss: 0.521 | Accuracy: 83.333\n",
      "Validation loss decreased (0.561772 --> 0.521174).  Saving model ...\n",
      "Validation loss decreased (0.521174 --> 0.484985).  Saving model ...\n",
      "Validation loss decreased (0.484985 --> 0.455635).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.455635 --> 0.432241).  Saving model ...\n",
      "Validation loss decreased (0.432241 --> 0.412974).  Saving model ...\n",
      "Validation loss decreased (0.412974 --> 0.396554).  Saving model ...\n",
      "Validation loss decreased (0.396554 --> 0.381492).  Saving model ...\n",
      "Validation loss decreased (0.381492 --> 0.368605).  Saving model ...\n",
      "Validation loss decreased (0.368605 --> 0.356482).  Saving model ...\n",
      "Validation loss decreased (0.356482 --> 0.346767).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.308 | Accuracy: 88.715\n",
      "Test Loss: 0.337 | Accuracy: 87.500\n",
      "Validation loss decreased (0.346767 --> 0.337409).  Saving model ...\n",
      "Validation loss decreased (0.337409 --> 0.329441).  Saving model ...\n",
      "Validation loss decreased (0.329441 --> 0.322673).  Saving model ...\n",
      "Validation loss decreased (0.322673 --> 0.317241).  Saving model ...\n",
      "Validation loss decreased (0.317241 --> 0.311371).  Saving model ...\n",
      "Validation loss decreased (0.311371 --> 0.305603).  Saving model ...\n",
      "Validation loss decreased (0.305603 --> 0.302066).  Saving model ...\n",
      "Validation loss decreased (0.302066 --> 0.298695).  Saving model ...\n",
      "Validation loss decreased (0.298695 --> 0.294418).  Saving model ...\n",
      "Validation loss decreased (0.294418 --> 0.290937).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.217 | Accuracy: 92.882\n",
      "Test Loss: 0.291 | Accuracy: 89.583\n",
      "Validation loss decreased (0.290937 --> 0.290706).  Saving model ...\n",
      "Validation loss decreased (0.290706 --> 0.285254).  Saving model ...\n",
      "Validation loss decreased (0.285254 --> 0.282599).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.282599 --> 0.278950).  Saving model ...\n",
      "Validation loss decreased (0.278950 --> 0.275609).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.275609 --> 0.272981).  Saving model ...\n",
      "Validation loss decreased (0.272981 --> 0.271272).  Saving model ...\n",
      "Validation loss decreased (0.271272 --> 0.270908).  Saving model ...\n",
      "Epoch: 40\n",
      "Train Loss: 0.150 | Accuracy: 95.139\n",
      "Test Loss: 0.269 | Accuracy: 90.972\n",
      "Validation loss decreased (0.270908 --> 0.268820).  Saving model ...\n",
      "Validation loss decreased (0.268820 --> 0.268361).  Saving model ...\n",
      "Validation loss decreased (0.268361 --> 0.267244).  Saving model ...\n",
      "Validation loss decreased (0.267244 --> 0.265680).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.265680 --> 0.263888).  Saving model ...\n",
      "Validation loss decreased (0.263888 --> 0.263640).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.263640 --> 0.263214).  Saving model ...\n",
      "Validation loss decreased (0.263214 --> 0.261775).  Saving model ...\n",
      "Epoch: 50\n",
      "Train Loss: 0.095 | Accuracy: 98.090\n",
      "Test Loss: 0.264 | Accuracy: 91.667\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Validation loss decreased (0.261775 --> 0.260659).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "Epoch: 60\n",
      "Train Loss: 0.060 | Accuracy: 99.132\n",
      "Test Loss: 0.269 | Accuracy: 92.361\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 5 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Linear(in_features=2098, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.682 | Accuracy: 83.333\n",
      "Test Loss: 0.680 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.679833).  Saving model ...\n",
      "Validation loss decreased (0.679833 --> 0.676870).  Saving model ...\n",
      "Validation loss decreased (0.676870 --> 0.673708).  Saving model ...\n",
      "Validation loss decreased (0.673708 --> 0.669613).  Saving model ...\n",
      "Validation loss decreased (0.669613 --> 0.664107).  Saving model ...\n",
      "Validation loss decreased (0.664107 --> 0.656546).  Saving model ...\n",
      "Validation loss decreased (0.656546 --> 0.646288).  Saving model ...\n",
      "Validation loss decreased (0.646288 --> 0.632904).  Saving model ...\n",
      "Validation loss decreased (0.632904 --> 0.615931).  Saving model ...\n",
      "Validation loss decreased (0.615931 --> 0.595460).  Saving model ...\n",
      "Epoch: 10\n",
      "Train Loss: 0.578 | Accuracy: 83.333\n",
      "Test Loss: 0.571 | Accuracy: 83.333\n",
      "Validation loss decreased (0.595460 --> 0.571300).  Saving model ...\n",
      "Validation loss decreased (0.571300 --> 0.543926).  Saving model ...\n",
      "Validation loss decreased (0.543926 --> 0.515633).  Saving model ...\n",
      "Validation loss decreased (0.515633 --> 0.487024).  Saving model ...\n",
      "Validation loss decreased (0.487024 --> 0.460760).  Saving model ...\n",
      "Validation loss decreased (0.460760 --> 0.437305).  Saving model ...\n",
      "Validation loss decreased (0.437305 --> 0.418386).  Saving model ...\n",
      "Validation loss decreased (0.418386 --> 0.401625).  Saving model ...\n",
      "Validation loss decreased (0.401625 --> 0.386591).  Saving model ...\n",
      "Validation loss decreased (0.386591 --> 0.374078).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.331 | Accuracy: 87.153\n",
      "Test Loss: 0.362 | Accuracy: 85.417\n",
      "Validation loss decreased (0.374078 --> 0.362393).  Saving model ...\n",
      "Validation loss decreased (0.362393 --> 0.352915).  Saving model ...\n",
      "Validation loss decreased (0.352915 --> 0.345252).  Saving model ...\n",
      "Validation loss decreased (0.345252 --> 0.337628).  Saving model ...\n",
      "Validation loss decreased (0.337628 --> 0.330361).  Saving model ...\n",
      "Validation loss decreased (0.330361 --> 0.324051).  Saving model ...\n",
      "Validation loss decreased (0.324051 --> 0.318248).  Saving model ...\n",
      "Validation loss decreased (0.318248 --> 0.312596).  Saving model ...\n",
      "Validation loss decreased (0.312596 --> 0.307228).  Saving model ...\n",
      "Validation loss decreased (0.307228 --> 0.302499).  Saving model ...\n",
      "Epoch: 30\n",
      "Train Loss: 0.230 | Accuracy: 92.882\n",
      "Test Loss: 0.298 | Accuracy: 90.278\n",
      "Validation loss decreased (0.302499 --> 0.298290).  Saving model ...\n",
      "Validation loss decreased (0.298290 --> 0.294290).  Saving model ...\n",
      "Validation loss decreased (0.294290 --> 0.290655).  Saving model ...\n",
      "Validation loss decreased (0.290655 --> 0.287103).  Saving model ...\n",
      "Validation loss decreased (0.287103 --> 0.283008).  Saving model ...\n",
      "Validation loss decreased (0.283008 --> 0.280210).  Saving model ...\n",
      "Validation loss decreased (0.280210 --> 0.276991).  Saving model ...\n",
      "Validation loss decreased (0.276991 --> 0.274781).  Saving model ...\n",
      "Validation loss decreased (0.274781 --> 0.271123).  Saving model ...\n",
      "Validation loss decreased (0.271123 --> 0.268452).  Saving model ...\n",
      "Epoch: 40\n",
      "Train Loss: 0.158 | Accuracy: 95.312\n",
      "Test Loss: 0.266 | Accuracy: 91.667\n",
      "Validation loss decreased (0.268452 --> 0.266170).  Saving model ...\n",
      "Validation loss decreased (0.266170 --> 0.263143).  Saving model ...\n",
      "Validation loss decreased (0.263143 --> 0.261098).  Saving model ...\n",
      "Validation loss decreased (0.261098 --> 0.259003).  Saving model ...\n",
      "Validation loss decreased (0.259003 --> 0.257182).  Saving model ...\n",
      "Validation loss decreased (0.257182 --> 0.254148).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.254148 --> 0.251413).  Saving model ...\n",
      "Validation loss decreased (0.251413 --> 0.249002).  Saving model ...\n",
      "Validation loss decreased (0.249002 --> 0.248033).  Saving model ...\n",
      "Epoch: 50\n",
      "Train Loss: 0.102 | Accuracy: 97.049\n",
      "Test Loss: 0.247 | Accuracy: 90.972\n",
      "Validation loss decreased (0.248033 --> 0.247096).  Saving model ...\n",
      "Validation loss decreased (0.247096 --> 0.245334).  Saving model ...\n",
      "Validation loss decreased (0.245334 --> 0.245182).  Saving model ...\n",
      "Validation loss decreased (0.245182 --> 0.242883).  Saving model ...\n",
      "Validation loss decreased (0.242883 --> 0.241896).  Saving model ...\n",
      "Validation loss decreased (0.241896 --> 0.240895).  Saving model ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss decreased (0.240895 --> 0.240521).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.240521 --> 0.239623).  Saving model ...\n",
      "Validation loss decreased (0.239623 --> 0.238675).  Saving model ...\n",
      "Epoch: 60\n",
      "Train Loss: 0.066 | Accuracy: 98.264\n",
      "Test Loss: 0.239 | Accuracy: 91.667\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.238675 --> 0.238110).  Saving model ...\n",
      "Validation loss decreased (0.238110 --> 0.237424).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.237424 --> 0.237280).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Epoch: 70\n",
      "Train Loss: 0.043 | Accuracy: 99.132\n",
      "Test Loss: 0.238 | Accuracy: 93.056\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "train_losses = []\n",
    "train_acc = []\n",
    "#test_acc = []\n",
    "test_losses= []\n",
    "metrics = []\n",
    "#for each fold\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    print('Split '+str(i+1)+' ......')\n",
    "    train_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/train_split'+str(i+1)+'.csv')\n",
    "    test_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/test_split'+str(i+1)+'.csv')\n",
    "    train_loader = data_process(train_set,32)\n",
    "    test_loader = data_process(test_set,len(test_set))\n",
    "    \n",
    "    model = GAT_MFP().float().to(device)\n",
    "    \n",
    "    model.apply(reset_weights)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=0.000075,weight_decay=0.001)\n",
    "    \n",
    "    #for each epoch\n",
    "    train_loss = []\n",
    "    test_loss = []\n",
    "    train_accuracy = []\n",
    "    test_accuracy = []\n",
    "    \n",
    "    early_stopping = EarlyStopping(patience=10, verbose=True)\n",
    "    #update the model parameters with 100 epochs\n",
    "    for epoch in range(100):\n",
    "        train(epoch,train_loader)\n",
    "        test(epoch,test_loader)\n",
    "        \n",
    "        early_stopping(test_loss[-1],model)\n",
    "        \n",
    "        if early_stopping.early_stop:\n",
    "            print('Early stopping')\n",
    "            break\n",
    "    \n",
    "    model.load_state_dict(torch.load('checkpoint.pt'))\n",
    "    auc, report = test_metrics(test_loader)\n",
    "    metric = [auc,report]\n",
    "    \n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    train_acc.append(train_accuracy)\n",
    "    test_losses.append(test_loss)\n",
    "    metrics.append(metric)\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.905 +/- 0.016831648705582826\n",
      "Accuracy: 0.9194444444444445 +/- 0.005555555555555585\n",
      "Precision: 0.9184939410196581 +/- 0.00719648532790548\n",
      "Recall: 0.9194444444444445 +/- 0.005555555555555585\n",
      "F1-score: 0.9122381693589142 +/- 0.0058301255005948115\n"
     ]
    }
   ],
   "source": [
    "print_metrics(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combination with all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class All(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(All, self).__init__()\n",
    "    \n",
    "        #before the Attention Mechanism, parse in a fully connected network and output a 50 dimensional vector \n",
    "        self.hidden = 50  \n",
    "        self.in_head = 5 #repeat the mechanism for 5 times\n",
    "        self.conv1 = GATConv(in_channels = 133, \n",
    "                             out_channels = self.hidden, \n",
    "                             heads=self.in_head, concat=False) #set concat to be False, so it will take average instead\n",
    "\n",
    "        \n",
    "        #define a transformer encoder layer to learn about smiles string\n",
    "        #vocab_len to decide how many unique 'word', 1024 the dimension of each embeddings\n",
    "        self.embedding = Embedding.from_pretrained(torch.from_numpy(wv.vectors),freeze=False)\n",
    "        \n",
    "        encoder_layer = TransformerEncoderLayer(d_model=1024, nhead=2)\n",
    "        \n",
    "        self.encoder = TransformerEncoder(encoder_layer,4)\n",
    "        self.pos_encoder = PositionalEncoding(1024, 0.1)\n",
    "        \n",
    "        #fully connected layers\n",
    "        self.linear1 = Linear(self.hidden+2048+1024,100)\n",
    "        self.linear2 = Linear(100, 25)\n",
    "        self.linear3 = Linear(25, 10)\n",
    "        self.linear4 = Linear(10, 1)\n",
    "\n",
    "    def forward(self, data):\n",
    "        \n",
    "        x, edge_index, batch_index, wv_index, mol_feature = data.x, data.edge_index, data.batch, data.emb, data.mol_feature        \n",
    "        #extract batched node and edge feature matrices, edge_index matrix, batch_index and token indices tensor \n",
    "        \n",
    "        wv_index = [torch.tensor(item) for item in wv_index]\n",
    "        \n",
    "        padded = pad_sequence(wv_index,padding_value = 266)\n",
    "        # seq_len x batch_size \n",
    "        \n",
    "        fp = self.embedding(padded.to(device))\n",
    "        #seq_len x batch_size x dim_embedding\n",
    "        \n",
    "        fp = self.pos_encoder(fp)\n",
    "        fp = self.encoder(fp)\n",
    "        #seq_len x batch_size x dim_embedding\n",
    "        #learned smile representation \n",
    "        \n",
    "        fp = fp.sum(dim=0)\n",
    "        x = self.conv1(x.float(), edge_index)\n",
    "\n",
    "        #aggregate the learned local node feature to capture the global property \n",
    "        #here, we apply global_mean_pool over the updated node feature\n",
    "        x = gap(x,batch_index)\n",
    "        \n",
    "        #also include some other global information i.e. binary morgan fingerprint\n",
    "        x = torch.cat([x, mol_feature.reshape(data.num_graphs,2048), fp],dim=1)\n",
    "\n",
    "        x = F.relu(self.linear1(x.float()))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = F.relu(self.linear3(x))\n",
    "        x = torch.sigmoid(self.linear4(x))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3122, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.647 | Accuracy: 79.514\n",
      "Test Loss: 0.443 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.443180).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.443180 --> 0.415381).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.328 | Accuracy: 87.674\n",
      "Test Loss: 0.979 | Accuracy: 45.139\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 2 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3122, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.573 | Accuracy: 80.382\n",
      "Test Loss: 0.963 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.962675).  Saving model ...\n",
      "Validation loss decreased (0.962675 --> 0.529997).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.529997 --> 0.473756).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.473756 --> 0.472557).  Saving model ...\n",
      "Validation loss decreased (0.472557 --> 0.363988).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.356 | Accuracy: 87.500\n",
      "Test Loss: 0.554 | Accuracy: 76.389\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 3 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3122, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n",
      "Train Loss: 0.523 | Accuracy: 83.681\n",
      "Test Loss: 0.686 | Accuracy: 84.028\n",
      "Validation loss decreased (inf --> 0.686269).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.686269 --> 0.484493).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Validation loss decreased (0.484493 --> 0.423754).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.312 | Accuracy: 88.542\n",
      "Test Loss: 0.400 | Accuracy: 85.417\n",
      "Validation loss decreased (0.423754 --> 0.400499).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "Validation loss decreased (0.400499 --> 0.393127).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "Epoch: 20\n",
      "Train Loss: 0.331 | Accuracy: 88.368\n",
      "Test Loss: 0.386 | Accuracy: 84.722\n",
      "Validation loss decreased (0.393127 --> 0.386105).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "Epoch: 30\n",
      "Train Loss: 0.207 | Accuracy: 93.403\n",
      "Test Loss: 0.822 | Accuracy: 87.500\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 4 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3122, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.584 | Accuracy: 79.340\n",
      "Test Loss: 0.508 | Accuracy: 82.639\n",
      "Validation loss decreased (inf --> 0.508494).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.508494 --> 0.361260).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.286 | Accuracy: 89.062\n",
      "Test Loss: 0.880 | Accuracy: 81.250\n",
      "EarlyStopping counter: 8 out of 10\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n",
      "Split 5 ......\n",
      "Reset trainable parameters of layer = Linear(133, 250, bias=False)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=1024, out_features=2048, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=2048, out_features=1024, bias=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "Reset trainable parameters of layer = GATConv(133, 50, heads=5)\n",
      "Reset trainable parameters of layer = Embedding(277, 1024)\n",
      "Reset trainable parameters of layer = Linear(in_features=3122, out_features=100, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=100, out_features=25, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=25, out_features=10, bias=True)\n",
      "Reset trainable parameters of layer = Linear(in_features=10, out_features=1, bias=True)\n",
      "Epoch: 0\n",
      "Train Loss: 0.531 | Accuracy: 83.160\n",
      "Test Loss: 0.588 | Accuracy: 83.333\n",
      "Validation loss decreased (inf --> 0.587902).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.587902 --> 0.429453).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "Validation loss decreased (0.429453 --> 0.366934).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "Epoch: 10\n",
      "Train Loss: 0.339 | Accuracy: 87.500\n",
      "Test Loss: 0.360 | Accuracy: 84.722\n",
      "Validation loss decreased (0.366934 --> 0.360122).  Saving model ...\n",
      "Validation loss decreased (0.360122 --> 0.333293).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "Validation loss decreased (0.333293 --> 0.311097).  Saving model ...\n",
      "Epoch: 20\n",
      "Train Loss: 0.278 | Accuracy: 89.583\n",
      "Test Loss: 0.343 | Accuracy: 88.194\n",
      "EarlyStopping counter: 1 out of 10\n",
      "Validation loss decreased (0.311097 --> 0.299175).  Saving model ...\n",
      "EarlyStopping counter: 1 out of 10\n",
      "EarlyStopping counter: 2 out of 10\n",
      "EarlyStopping counter: 3 out of 10\n",
      "EarlyStopping counter: 4 out of 10\n",
      "EarlyStopping counter: 5 out of 10\n",
      "EarlyStopping counter: 6 out of 10\n",
      "EarlyStopping counter: 7 out of 10\n",
      "EarlyStopping counter: 8 out of 10\n",
      "Epoch: 30\n",
      "Train Loss: 0.229 | Accuracy: 89.931\n",
      "Test Loss: 0.482 | Accuracy: 80.556\n",
      "EarlyStopping counter: 9 out of 10\n",
      "EarlyStopping counter: 10 out of 10\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "train_losses = []\n",
    "train_acc = []\n",
    "#test_acc = []\n",
    "test_losses= []\n",
    "metrics = []\n",
    "#for each fold\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    print('Split '+str(i+1)+' ......')\n",
    "    train_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/train_split'+str(i+1)+'.csv')\n",
    "    test_set = pd.read_csv('dir_to_dataset/5_fold_cv_train_test/test_split'+str(i+1)+'.csv')\n",
    "    train_loader = data_process(train_set,6)\n",
    "    test_loader = data_process(test_set,len(test_set))\n",
    "    \n",
    "    model = All().float().to(device)\n",
    "    \n",
    "    model.apply(reset_weights)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=0.000075,weight_decay=0.001)\n",
    "    \n",
    "    #for each epoch\n",
    "    train_loss = []\n",
    "    test_loss = []\n",
    "    train_accuracy = []\n",
    "    test_accuracy = []\n",
    "    \n",
    "    early_stopping = EarlyStopping(patience=10, verbose=True)\n",
    "    #update the model parameters with 100 epochs\n",
    "    for epoch in range(100):\n",
    "        train(epoch,train_loader)\n",
    "        test(epoch,test_loader)\n",
    "        \n",
    "        early_stopping(test_loss[-1],model)\n",
    "        \n",
    "        if early_stopping.early_stop:\n",
    "            print('Early stopping')\n",
    "            break\n",
    "    \n",
    "    model.load_state_dict(torch.load('checkpoint.pt'))\n",
    "    auc, report = test_metrics(test_loader)\n",
    "    metric = [auc,report]\n",
    "    \n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    train_acc.append(train_accuracy)\n",
    "    test_losses.append(test_loss)\n",
    "    metrics.append(metric)\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.8186805555555555 +/- 0.03564834355405802\n",
      "Accuracy: 0.8569444444444445 +/- 0.01734721666221776\n",
      "Precision: 0.8532421823194486 +/- 0.020276658095891555\n",
      "Recall: 0.8569444444444445 +/- 0.01734721666221776\n",
      "F1-score: 0.8340719141374023 +/- 0.03695717136018627\n"
     ]
    }
   ],
   "source": [
    "print_metrics(metrics)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}