Part14_ClassificationAfterSampling_ML.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b42f5d50",
   "metadata": {},
   "source": [
    "# Classification after Sampling Part 1 (SVM, MLPClassifier, KNN, Randon Forest) on Vector Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "529fa14d",
   "metadata": {},
   "source": [
    "### Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "255d0498",
   "metadata": {},
   "outputs": [],
   "source": [
    "# float(\"{:.2f}\".format(13.949999999999999))\n",
    "\n",
    "def TSS(TP,TN,FP,FN):\n",
    "    TSS_value = (TP / (TP + FN)) - (FP / (FP + TN))\n",
    "    return TSS_value\n",
    "\n",
    "def HSS1(TP,TN,FP,FN):\n",
    "    HSS1_value = (2 * (TP * TN - FP * FN)) / ((TP + FN) * (FN + TN) + (TP + FP) * (FP + TN))\n",
    "    return HSS1_value\n",
    "    \n",
    "def HSS2(TP,TN,FP,FN):\n",
    "    HSS2_value = (2 * (TP * TN - FP * FN)) / ((TP + FP) * (FN + TN) + (TP + FN) * (FP + TN))\n",
    "    return HSS2_value\n",
    "\n",
    "def GSS(TP,TN,FP,FN):\n",
    "    GSS_value = (TP - (TP + FP) * (TP + FN) / (TP + FP + FN + TN))\n",
    "    return GSS_value\n",
    "\n",
    "def Recall(TP,TN,FP,FN):\n",
    "    Recall_value = (TP) / (TP + FN)\n",
    "    return Recall_value\n",
    "\n",
    "def FPR(TP,TN,FP,FN):\n",
    "    fpr_value = (FP) / (FP + TN)\n",
    "    return fpr_value\n",
    "\n",
    "def Accuracy(TP,TN,FP,FN):\n",
    "    accuracy_value = (TP + TN) / (TP + TN + FP + FN)\n",
    "    return accuracy_value\n",
    "\n",
    "def Precision(TP,TN,FP,FN):\n",
    "    precision_value = (FP) / (TP + FP)\n",
    "    return precision_value"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d43af151",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation and Without C Class and RUS-TomekLinks-SMOTE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "06c2f0ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/8_1_FinalData_OUSampling_NewFeatures_LSBZM_KnnImputation/\"\n",
    "X_train_NewF_ZM_RTS = []\n",
    "Y_train_NewF_ZM_RTS = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"RUS_Tomek_Smote_\" +\"Partition\" + str(i+1) + \"_OUSampling_WithoutC_NewFeatures_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        X_train_NewF_ZM_RTS.append(pickle.load(f))\n",
    "    with open(data_dir + \"RUS_Tomek_Smote_\" + \"Partition\" + str(i+1) + \"_Labels_OUSampling_WithoutC_NewFeatures_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        Y_train_NewF_ZM_RTS.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_train_NewF_ZM_RTS[i]).any() or np.isinf(X_train_NewF_ZM_RTS[i]).any()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c505b1bd",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation and Without C Class and RUS-TomekLinks-ADASYN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ce5553a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/8_1_FinalData_OUSampling_NewFeatures_LSBZM_KnnImputation/\"\n",
    "X_train_NewF_ZM_RTA = []\n",
    "Y_train_NewF_ZM_RTA = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"RUS_Tomek_Adasyn_\" +\"Partition\" + str(i+1) + \"_OUSampling_WithoutC_NewFeatures_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        X_train_NewF_ZM_RTA.append(pickle.load(f))\n",
    "    with open(data_dir + \"RUS_Tomek_Adasyn_\" + \"Partition\" + str(i+1) + \"_Labels_OUSampling_WithoutC_NewFeatures_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        Y_train_NewF_ZM_RTA.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_train_NewF_ZM_RTA[i]).any() or np.isinf(X_train_NewF_ZM_RTA[i]).any()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afe4fdc6",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation and Without C Class and SMOTE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d5e9ba63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/8_2_FinalData_OSampling_NewFeatures_LSBZM_KnnImputation/\"\n",
    "X_train_NewF_ZM_S = []\n",
    "Y_train_NewF_ZM_S = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"Smote_\" +\"Partition\" + str(i+1) + \"_OSampling_WithoutC_NewFeatures_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        X_train_NewF_ZM_S.append(pickle.load(f))\n",
    "    with open(data_dir + \"Smote_\" + \"Partition\" + str(i+1) + \"_Labels_OSampling_WithoutC_NewFeatures_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        Y_train_NewF_ZM_S.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_train_NewF_ZM_S[i]).any() or np.isinf(X_train_NewF_ZM_S[i]).any()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e35efd9",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation and Without C Class and ADASYN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1dd52129",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/8_2_FinalData_OSampling_NewFeatures_LSBZM_KnnImputation/\"\n",
    "X_train_NewF_ZM_A = []\n",
    "Y_train_NewF_ZM_A = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"Adasyn_\" +\"Partition\" + str(i+1) + \"_OSampling_WithoutC_NewFeatures_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        X_train_NewF_ZM_A.append(pickle.load(f))\n",
    "    with open(data_dir + \"Adasyn_\" + \"Partition\" + str(i+1) + \"_Labels_OSampling_WithoutC_NewFeatures_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        Y_train_NewF_ZM_A.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_train_NewF_ZM_A[i]).any() or np.isinf(X_train_NewF_ZM_A[i]).any()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4411f8cb",
   "metadata": {},
   "source": [
    "### Test Data- Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bfdd3940",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_2_FinalData_NewFeatures_LSBZM_KnnImputation/\"\n",
    "X_test_NewF_ZM = []\n",
    "Y_test_NewF_ZM = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_NewFeatures_LSBZM_KnnImputation\" + \".pkl\", 'rb') as f:\n",
    "        X_test_NewF_ZM.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_test_NewF_ZM[i]).any() or np.isinf(X_test_NewF_ZM[i]).any()))\n",
    "    \n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_NewFeatures_LSBZM_KnnImputation\" + \".pkl\", 'rb') as f:\n",
    "        Y_test_NewF_ZM.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d520699a",
   "metadata": {},
   "source": [
    "### Function to run the classification on different Train-Test Splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ed6ceb4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def kfold_training(name, X_train, Y_train, X_test,Y_test, training_func, num):\n",
    "    #kfold = np.array([[1,2],[1,3],[1,4],[1,5],[2,3],[2,4],[2,5],[3,4],[3,5],[4,5]])\n",
    "    kfold = np.array([[1,2],[2,3],[3,4],[4,5]])\n",
    "    metrics = []\n",
    "    metrics_values = np.array([])\n",
    "    \n",
    "    for i in range(0, num):\n",
    "        train_index = kfold[i,0]\n",
    "        test_index = kfold[i,1]\n",
    "        metrics_values = training_func(X_train[train_index-1], Y_train[train_index-1], X_test[test_index-1], Y_test[test_index-1])\n",
    "        while (metrics_values[0] == 0):\n",
    "            metrics_values = training_func(X_train[train_index-1], Y_train[train_index-1], X_test[test_index-1], Y_test[test_index-1])\n",
    "        \n",
    "        metrics.append(np.append(np.append(train_index, test_index), metrics_values))\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "faf5130c",
   "metadata": {},
   "source": [
    "## Code of the Classification techniques"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10454d6f",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9a1291ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary libraries\n",
    "import numpy as np\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def svm_model(X_train, Y_train, X_test, Y_test):\n",
    "    \n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/\"\n",
    "\n",
    "\n",
    "    # Create an SVM classifier (you can choose different kernels like 'linear', 'rbf', etc.)\n",
    "    svm_classifier = SVC(kernel='rbf', C=1.0)\n",
    "    svm_classifier.fit(X_train, Y_train)\n",
    "    y_pred = svm_classifier.predict(X_test)\n",
    "    \n",
    "    \n",
    "    print(str(X_train.shape)+': SVM Classifier is Done! \\n')\n",
    "    \n",
    "\n",
    "    confusion = confusion_matrix(Y_test, y_pred)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp,tn,fp,fn)\n",
    "    hss1 = HSS1(tp,tn,fp,fn)\n",
    "    hss2 = HSS2(tp,tn,fp,fn)\n",
    "    gss = GSS(tp,tn,fp,fn)\n",
    "    recall = Recall(tp,tn,fp,fn)\n",
    "    precision = Precision(tp,tn,fp,fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "\n",
    "    #joblib.dump(classifier, data_dir + \"svm_model.pkl\")\n",
    "\n",
    "    #loaded_svm_model = joblib.load(data_dir + \"svm_model.pkl\")\n",
    "    \n",
    "    return output_values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12a9a3ae",
   "metadata": {},
   "source": [
    "### MPL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a780f706",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def mlp_model(X_train, Y_train, X_test, Y_test):\n",
    "    \n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/\"\n",
    "\n",
    "    # Define the MLP model\n",
    "    # Define the MLP model with four hidden layers\n",
    "    model = keras.Sequential([\n",
    "        layers.Input(shape=(216,)),  # Input layer with 216 features\n",
    "        layers.Dense(64, activation='relu'),  # Hidden layer with 64 units and ReLU activation\n",
    "        layers.Dense(32, activation='relu'),  # Hidden layer with 32 units and ReLU activation\n",
    "        layers.Dense(16, activation='relu'),  # Hidden layer with 16 units and ReLU activation\n",
    "        layers.Dense(8, activation='relu'),  # Hidden layer with 16 units and ReLU activation\n",
    "        layers.Dense(1, activation='sigmoid')  # Output layer with 1 unit and sigmoid activation (binary classification)\n",
    "    ])\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=keras.metrics.SpecificityAtSensitivity(sensitivity=0.95))\n",
    "\n",
    "    # Train the model\n",
    "    model.fit(X_train, Y_train, epochs=15, batch_size=32, verbose=0)  # Adjust epochs and batch_size as needed\n",
    "\n",
    "    best_threshold = 0.0\n",
    "    best_tss = 0.0\n",
    "    y_pred = model.predict(X_test)\n",
    "    # evaluate model\n",
    "    for i in range(1, 1000):\n",
    "\n",
    "        threshold = i / 1000 # Adjust the threshold as needed\n",
    "        y_pred_binary = (y_pred > threshold).astype(int)\n",
    "        confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "        tn, fp, fn, tp = confusion.ravel()\n",
    "        tss = TSS(tp,tn,fp,fn)\n",
    "        if tss > best_tss:\n",
    "            best_tss = tss\n",
    "            best_threshold = i / 1000\n",
    "        \n",
    "    \n",
    "    print(str(X_train.shape)+': MLP Classifier is Done! \\n')\n",
    "\n",
    "\n",
    "    threshold = best_threshold # Adjust the threshold as needed\n",
    "    y_pred_binary = (y_pred > threshold).astype(int)\n",
    "    confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp,tn,fp,fn)\n",
    "    hss1 = HSS1(tp,tn,fp,fn)\n",
    "    hss2 = HSS2(tp,tn,fp,fn)\n",
    "    gss = GSS(tp,tn,fp,fn)\n",
    "    recall = Recall(tp,tn,fp,fn)\n",
    "    precision = Precision(tp,tn,fp,fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "\n",
    "    #joblib.dump(classifier, data_dir + \"mlp_model.pkl\")\n",
    "\n",
    "    #loaded_mlp_model = joblib.load(data_dir + \"mlp_model.pkl\")\n",
    "    \n",
    "    return output_values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8903b60",
   "metadata": {},
   "source": [
    "### k-NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0507e01b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def knn_model(X_train, Y_train, X_test, Y_test):\n",
    "    \n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/\"\n",
    "\n",
    "    # Create a Gaussian Naive Bayes classifier\n",
    "    nb_classifier = KNeighborsClassifier(n_neighbors=10)\n",
    "    nb_classifier.fit(X_train, Y_train)\n",
    "    y_pred = nb_classifier.predict(X_test)\n",
    "    \n",
    "    print(str(X_train.shape) + ': KNeighborsClassifier Classifier is Done! \\n')\n",
    "\n",
    "    confusion = confusion_matrix(Y_test, y_pred)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp,tn,fp,fn)\n",
    "    hss1 = HSS1(tp,tn,fp,fn)\n",
    "    hss2 = HSS2(tp,tn,fp,fn)\n",
    "    gss = GSS(tp,tn,fp,fn)\n",
    "    recall = Recall(tp,tn,fp,fn)\n",
    "    precision = Precision(tp,tn,fp,fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "    #joblib.dump(nb_classifier, data_dir + \"naive_bayes_model.pkl\")\n",
    "\n",
    "    #loaded_nb_model = joblib.load(data_dir + \"naive_bayes_model.pkl\")\n",
    "\n",
    "    return output_values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8be28f51",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d4249dc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def random_forest_model(X_train, Y_train, X_test, Y_test):\n",
    "    \n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/\"\n",
    "\n",
    "    # Create a Random Forest classifier\n",
    "    # You can adjust 'n_estimators' and other parameters as needed\n",
    "    rf_classifier = RandomForestClassifier(n_estimators=60, random_state=42, criterion='entropy')\n",
    "    rf_classifier.fit(X_train, Y_train)\n",
    "    y_pred = rf_classifier.predict(X_test)\n",
    "    \n",
    "    print(str(X_train.shape) + ': Random Forest Classifier is Done! \\n')\n",
    "\n",
    "    confusion = confusion_matrix(Y_test, y_pred)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp, tn, fp, fn)\n",
    "    hss1 = HSS1(tp, tn, fp, fn)\n",
    "    hss2 = HSS2(tp, tn, fp, fn)\n",
    "    gss = GSS(tp, tn, fp, fn)\n",
    "    recall = Recall(tp, tn, fp, fn)\n",
    "    precision = Precision(tp, tn, fp, fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "    #joblib.dump(rf_classifier, data_dir + \"random_forest_model.pkl\")\n",
    "\n",
    "    #loaded_rf_model = joblib.load(data_dir + \"random_forest_model.pkl\")\n",
    "\n",
    "    return output_values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32e9fd1e",
   "metadata": {},
   "source": [
    "### Saving the Result of Classification on File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "127986e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_results(reslut, name):\n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/results/\"\n",
    "\n",
    "    with open(data_dir + name + \".pkl\", 'wb') as f:\n",
    "        pickle.dump(reslut, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a50d002e",
   "metadata": {},
   "source": [
    "## Running Experiments "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c7bc689",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5b280418",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19996, 216): SVM Classifier is Done! \n",
      "\n",
      "(19998, 216): SVM Classifier is Done! \n",
      "\n",
      "(19994, 216): SVM Classifier is Done! \n",
      "\n",
      "(19996, 216): SVM Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# SVM NewFeatures\n",
    "svm_newf_RTS = kfold_training('SVM', X_train_NewF_ZM_RTS, Y_train_NewF_ZM_RTS, X_test_NewF_ZM, Y_test_NewF_ZM, svm_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a14a0c88",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(svm_newf_RTS, \"SVM_RusTomekSmote_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d472edb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20061, 216): SVM Classifier is Done! \n",
      "\n",
      "(20043, 216): SVM Classifier is Done! \n",
      "\n",
      "(19955, 216): SVM Classifier is Done! \n",
      "\n",
      "(19981, 216): SVM Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "svm_newf_RTA = kfold_training('SVM', X_train_NewF_ZM_RTA, Y_train_NewF_ZM_RTA, X_test_NewF_ZM, Y_test_NewF_ZM, svm_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6a9b372f",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(svm_newf_RTA, \"SVM_RusTomekAdasyn_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "112f5152",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(131644, 216): SVM Classifier is Done! \n",
      "\n",
      "(156692, 216): SVM Classifier is Done! \n",
      "\n",
      "(70894, 216): SVM Classifier is Done! \n",
      "\n",
      "(88280, 216): SVM Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "svm_newf_S = kfold_training('SVM', X_train_NewF_ZM_S, Y_train_NewF_ZM_S, X_test_NewF_ZM, Y_test_NewF_ZM, svm_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "60b70408",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(svm_newf_S, \"SVM_Smote_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c419d067",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(131540, 216): SVM Classifier is Done! \n",
      "\n",
      "(156622, 216): SVM Classifier is Done! \n",
      "\n",
      "(70907, 216): SVM Classifier is Done! \n",
      "\n",
      "(88334, 216): SVM Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "svm_newf_A = kfold_training('SVM', X_train_NewF_ZM_A, Y_train_NewF_ZM_A, X_test_NewF_ZM, Y_test_NewF_ZM, svm_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8a862309",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(svm_newf_A, \"SVM_Adasyn_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5ce17de",
   "metadata": {},
   "source": [
    "### MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4dd95319",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 1s 242us/step\n",
      "(19996, 216): MLP Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 0s 240us/step\n",
      "(19998, 216): MLP Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 0s 223us/step\n",
      "(19994, 216): MLP Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 1s 224us/step\n",
      "(19996, 216): MLP Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# MLPClassifier NewFeatures\n",
    "mlp_newf_RTS = kfold_training('MLP', X_train_NewF_ZM_RTS, Y_train_NewF_ZM_RTS, X_test_NewF_ZM, Y_test_NewF_ZM, mlp_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "097d194e",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(mlp_newf_RTS, \"MLPClassifier_RusTomekSmote_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ea08e757",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 1s 223us/step\n",
      "(20061, 216): MLP Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 0s 231us/step\n",
      "(20043, 216): MLP Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 0s 223us/step\n",
      "(19955, 216): MLP Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 1s 237us/step\n",
      "(19981, 216): MLP Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "mlp_newf_RTA = kfold_training('MLP', X_train_NewF_ZM_RTA, Y_train_NewF_ZM_RTA, X_test_NewF_ZM, Y_test_NewF_ZM, mlp_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "27e9831d",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(mlp_newf_RTA, \"MLPClassifier_RusTomekAdasyn_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ffa8cae3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 1s 221us/step\n",
      "(131644, 216): MLP Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 0s 226us/step\n",
      "(156692, 216): MLP Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 0s 226us/step\n",
      "(70894, 216): MLP Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 1s 225us/step\n",
      "(88280, 216): MLP Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "mlp_newf_S = kfold_training('MLP', X_train_NewF_ZM_S, Y_train_NewF_ZM_S, X_test_NewF_ZM, Y_test_NewF_ZM, mlp_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8a38308a",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(mlp_newf_S, \"MLPClassifier_Smote_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f2628e64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 1s 225us/step\n",
      "(131540, 216): MLP Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 0s 225us/step\n",
      "(156622, 216): MLP Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 0s 227us/step\n",
      "(70907, 216): MLP Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 1s 225us/step\n",
      "(88334, 216): MLP Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "mlp_newf_A = kfold_training('MLP', X_train_NewF_ZM_A, Y_train_NewF_ZM_A, X_test_NewF_ZM, Y_test_NewF_ZM, mlp_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ebfee2b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(mlp_newf_A, \"MLPClassifier_Adasyn_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e702be07",
   "metadata": {},
   "source": [
    "### k-NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b24408d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19996, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(19998, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(19994, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(19996, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "naive_newf_RTS = kfold_training('KNN', X_train_NewF_ZM_RTS, Y_train_NewF_ZM_RTS, X_test_NewF_ZM, Y_test_NewF_ZM, knn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6ac1c1a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(naive_newf_RTS, \"KNN_RusTomekSmote_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7d9d7ab6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20061, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(20043, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(19955, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(19981, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "naive_newf_RTA = kfold_training('KNN', X_train_NewF_ZM_RTA, Y_train_NewF_ZM_RTA, X_test_NewF_ZM, Y_test_NewF_ZM, knn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f737eec5",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(naive_newf_RTA, \"KNN_RusTomekAdasyn_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "035da67f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(131644, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(156692, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(70894, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(88280, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "naive_newf_S = kfold_training('KNN', X_train_NewF_ZM_S, Y_train_NewF_ZM_S, X_test_NewF_ZM, Y_test_NewF_ZM, knn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "dff7f99d",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(naive_newf_S, \"KNN_Smote_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d2cb86a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(131540, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(156622, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(70907, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n",
      "(88334, 216): KNeighborsClassifier Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "naive_newf_A = kfold_training('KNN', X_train_NewF_ZM_A, Y_train_NewF_ZM_A, X_test_NewF_ZM, Y_test_NewF_ZM, knn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7d9ac7b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(naive_newf_A, \"KNN_Adasyn_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c43112c",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "fc2f461e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19996, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(19998, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(19994, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(19996, 216): Random Forest Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf_newf_RTS = kfold_training('RandomForest', X_train_NewF_ZM_RTS, Y_train_NewF_ZM_RTS, X_test_NewF_ZM, Y_test_NewF_ZM, random_forest_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5b25573a",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(rf_newf_RTS, \"RandomForest_RusTomekSmote_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "643a09dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20061, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(20043, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(19955, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(19981, 216): Random Forest Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf_newf_RTA = kfold_training('RandomForest', X_train_NewF_ZM_RTA, Y_train_NewF_ZM_RTA, X_test_NewF_ZM, Y_test_NewF_ZM, random_forest_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "186bd6a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(rf_newf_RTA, \"RandomForest_RusTomekAdasyn_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "bbfd510c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(131644, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(156692, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(70894, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(88280, 216): Random Forest Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf_newf_S = kfold_training('RandomForest', X_train_NewF_ZM_S, Y_train_NewF_ZM_S, X_test_NewF_ZM, Y_test_NewF_ZM, random_forest_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "92b85a0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(rf_newf_S, \"RandomForest_Smote_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "25b30ef1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(131540, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(156622, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(70907, 216): Random Forest Classifier is Done! \n",
      "\n",
      "(88334, 216): Random Forest Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf_newf_A = kfold_training('RandomForest', X_train_NewF_ZM_A, Y_train_NewF_ZM_A, X_test_NewF_ZM, Y_test_NewF_ZM, random_forest_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7b699559",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(rf_newf_A, \"RandomForest_Adasyn_WithoutC_NewFeatures_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db226f87",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}