coughit.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "coughit.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "iKTM4QzPSEZM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1202a608-0554-4df1-fe7f-1163a9254efd"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YVDDi4NR3EuB"
      },
      "source": [
        "!pip install tensorflow==2.4.3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install scikeras"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n_UeqsszOY0L",
        "outputId": "6d0bbee6-cc6a-44d9-d815-aebd6f683733"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting scikeras\n",
            "  Downloading scikeras-0.6.0-py3-none-any.whl (27 kB)\n",
            "Requirement already satisfied: packaging<22.0,>=0.21 in /usr/local/lib/python3.7/dist-packages (from scikeras) (21.3)\n",
            "Requirement already satisfied: scikit-learn>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from scikeras) (1.0.1)\n",
            "Collecting importlib-metadata<4,>=3\n",
            "  Downloading importlib_metadata-3.10.1-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata<4,>=3->scikeras) (3.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata<4,>=3->scikeras) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging<22.0,>=0.21->scikeras) (3.0.6)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=1.0.0->scikeras) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.14.6 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=1.0.0->scikeras) (1.19.5)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=1.0.0->scikeras) (1.4.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=1.0.0->scikeras) (3.0.0)\n",
            "Installing collected packages: importlib-metadata, scikeras\n",
            "  Attempting uninstall: importlib-metadata\n",
            "    Found existing installation: importlib-metadata 4.8.2\n",
            "    Uninstalling importlib-metadata-4.8.2:\n",
            "      Successfully uninstalled importlib-metadata-4.8.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "markdown 3.3.6 requires importlib-metadata>=4.4; python_version < \"3.10\", but you have importlib-metadata 3.10.1 which is incompatible.\u001b[0m\n",
            "Successfully installed importlib-metadata-3.10.1 scikeras-0.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wAX7vm8I3Gsf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "74179819-818d-405a-8c8c-d246c697c0d7"
      },
      "source": [
        "import tensorflow\n",
        "tensorflow.__version__"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'2.4.3'"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vAVMJgUfSw1L"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import json\n",
        "\n",
        "import librosa\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D, GlobalAveragePooling2D, BatchNormalization\n",
        "from tensorflow.keras.models import Sequential, load_model\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras import regularizers\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.model_selection import cross_val_score\n",
        "# from tensorflow.keras.wrappers.scikit_learn import KerasClassifier\n",
        "from scikeras.wrappers import KerasClassifier\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b2hltRAkS5Jv"
      },
      "source": [
        "data_path = '/content/gdrive/MyDrive/COVID_cough/mfcc15_data_audiomentations.json'\n",
        "test_data_path = '/content/gdrive/MyDrive/mfcc15_blind_set.json'\n",
        "test_metadata_path = '/content/gdrive/MyDrive/COVID_cough/test_metadata.csv'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cJ97cKAyS9CT"
      },
      "source": [
        "with open(data_path, \"r\") as fp:\n",
        "    data = json.load(fp)\n",
        "        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RPHixARfS-lA"
      },
      "source": [
        "X = np.array(data[\"mfcc\"])\n",
        "y = np.array(data[\"label\"])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oqzOK737TBYx",
        "outputId": "865ac08f-e4fc-4f27-fb93-b8fa460b4a83"
      },
      "source": [
        "print(X.shape)\n",
        "print(y.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(1280, 302, 15)\n",
            "(1280,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Le8X8E-1VGKf"
      },
      "source": [
        "X=X.reshape(X.shape[0],-1,15,1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nnSX6DdNS7jD",
        "outputId": "e8bce857-6392-4c50-8dd2-132931833ef2"
      },
      "source": [
        "print(X.shape)\n",
        "print(y.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(1280, 302, 15, 1)\n",
            "(1280,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5RfAZhjsXOvt"
      },
      "source": [
        "# https://www.kite.com/python/answers/how-to-serialize-a-numpy-array-into-json-in-python\n",
        "X[0].tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TLhg2t-DTGBP"
      },
      "source": [
        "def create_model():\n",
        "    model=Sequential()\n",
        "\n",
        "    model.add(Conv2D(filters=64,kernel_size=(3,3),input_shape=(302,15,1),activation='relu'))\n",
        "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "    model.add(Conv2D(filters=32,kernel_size=(2,2),activation='relu'))\n",
        "    model.add(BatchNormalization())\n",
        "    model.add(Flatten())\n",
        "\n",
        "    model.add(Dense(256,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=3e-4, l2=3e-3),bias_regularizer=regularizers.l2(3e-3),activity_regularizer=regularizers.l2(3e-4)))\n",
        "    model.add(Dropout(0.5))\n",
        "    model.add(Dense(128,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-3, l2=1e-2),bias_regularizer=regularizers.l2(1e-2),activity_regularizer=regularizers.l2(1e-3)))\n",
        "    model.add(Dropout(0.3))\n",
        "    model.add(Dense(1,activation='sigmoid'))\n",
        "\n",
        "    model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.0001), loss='binary_crossentropy', metrics=['accuracy'])\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# model_cv = KerasClassifier(build_fn=create_model, epochs=200, batch_size=32)\n",
        "# model_cv_200 = KerasClassifier(build_fn=create_model, epochs=200, batch_size=32)\n",
        "model_cv_20 = KerasClassifier(build_fn=create_model, epochs=20, batch_size=32)\n",
        "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)\n",
        "# scores_200 = cross_val_score(model_cv_200, X, y, cv= kfold, n_jobs=-1)\n",
        "scores_20 = cross_val_score(model_cv_20, X, y, cv= kfold, n_jobs=-1)\n",
        "# scores_200, \n",
        "scores_20"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Bbp3HDuPTCc",
        "outputId": "5494a2a2-0927-470b-ba34-224d780fbc14"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.8828125 , 0.8984375 , 0.92578125, 0.92578125, 0.87890625])"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# model_200 = create_model()\n",
        "# model_200.fit(X, y, epochs=200, batch_size=32)\n",
        "model_200 = load_model('/content/gdrive/MyDrive/cough-correct-model.h5')"
      ],
      "metadata": {
        "id": "0yp6OXy7S1rT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_20 = create_model()\n",
        "model_20.fit(X, y, epochs=20, batch_size=32)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w0gmgp9VdyII",
        "outputId": "4ba1716a-11ea-4353-f95c-87597516efc2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n",
            "40/40 [==============================] - 4s 19ms/step - loss: 19.5791 - accuracy: 0.7102\n",
            "Epoch 2/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 17.2598 - accuracy: 0.7820\n",
            "Epoch 3/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 15.0280 - accuracy: 0.8180\n",
            "Epoch 4/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 12.9902 - accuracy: 0.8727\n",
            "Epoch 5/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 11.2393 - accuracy: 0.9117\n",
            "Epoch 6/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 9.7731 - accuracy: 0.9414\n",
            "Epoch 7/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 8.5199 - accuracy: 0.9531\n",
            "Epoch 8/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 7.4788 - accuracy: 0.9750\n",
            "Epoch 9/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 6.6042 - accuracy: 0.9844\n",
            "Epoch 10/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 5.8449 - accuracy: 0.9812\n",
            "Epoch 11/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 5.2126 - accuracy: 0.9898\n",
            "Epoch 12/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 4.6535 - accuracy: 0.9961\n",
            "Epoch 13/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 4.1675 - accuracy: 0.9977\n",
            "Epoch 14/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 3.7555 - accuracy: 0.9969\n",
            "Epoch 15/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 3.3881 - accuracy: 0.9969\n",
            "Epoch 16/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 3.0806 - accuracy: 0.9961\n",
            "Epoch 17/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 2.7973 - accuracy: 0.9984\n",
            "Epoch 18/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 2.5538 - accuracy: 0.9953\n",
            "Epoch 19/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 2.3161 - accuracy: 0.9984\n",
            "Epoch 20/20\n",
            "40/40 [==============================] - 1s 18ms/step - loss: 2.1104 - accuracy: 0.9984\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f4110515c90>"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j9ha5D0B72AQ"
      },
      "source": [
        "fold_dict={}\n",
        "fold_dict['y_test']=y[test]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "di83ws32TMud",
        "outputId": "6922c275-02cd-4253-ed90-9446af78c546"
      },
      "source": [
        "from sklearn.model_selection import StratifiedKFold\n",
        "\n",
        "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)\n",
        "\n",
        "pred_folds_list=[]\n",
        "\n",
        "cvscores = []\n",
        "for train, test in kfold.split(X, y):\n",
        "    modell=Sequential()\n",
        "\n",
        "    modell.add(Conv2D(filters=64,kernel_size=(3,3),input_shape=(302,15,1),activation='relu'))\n",
        "    modell.add(MaxPooling2D(pool_size=(2,2)))\n",
        "    modell.add(Conv2D(filters=32,kernel_size=(2,2),activation='relu'))\n",
        "    modell.add(BatchNormalization())\n",
        "    modell.add(Flatten())\n",
        "\n",
        "    modell.add(Dense(256,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=3e-4, l2=3e-3),bias_regularizer=regularizers.l2(3e-3),activity_regularizer=regularizers.l2(3e-4)))\n",
        "    modell.add(Dropout(0.5))\n",
        "    modell.add(Dense(128,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-3, l2=1e-2),bias_regularizer=regularizers.l2(1e-2),activity_regularizer=regularizers.l2(1e-3)))\n",
        "    modell.add(Dropout(0.3))\n",
        "    modell.add(Dense(1,activation='sigmoid'))\n",
        "\n",
        "    modell.compile(optimizer=optimiser,\n",
        "                  loss='binary_crossentropy',\n",
        "                  metrics=['accuracy'])\n",
        "\n",
        "    fold_dict=dict()\n",
        "    modell.fit(X[train], y[train], epochs=200, batch_size=32, verbose=0)\n",
        "    # evaluate the model\n",
        "    scores = modell.evaluate( X[test], y[test], verbose=0)\n",
        "    fold_dict['y_test']=y[test]\n",
        "    fold_dict['y_pred']=modell.predict(X[test])\n",
        "    pred_folds_list.append(fold_dict)\n",
        "    \n",
        "    print(\"%s: %.2f%%\" % (modell.metrics_names[1], scores[1]*100))\n",
        "    cvscores.append(scores[1] * 100)\n",
        " \n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy: 87.50%\n",
            "accuracy: 89.84%\n",
            "accuracy: 91.02%\n",
            "accuracy: 90.23%\n",
            "accuracy: 91.41%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Uv-ufURUvNe",
        "outputId": "66b0abf8-962d-4210-a209-66ed3ca23449"
      },
      "source": [
        "\n",
        "print(\"%.2f%% (+/- %.2f%%)\" % (np.mean(cvscores), np.std(cvscores)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "92.66% (+/- 4.22%)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o9YvxineUQgf"
      },
      "source": [
        " history = model.fit(X, y, batch_size=32, epochs=150)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# blind "
      ],
      "metadata": {
        "id": "9hby9cIMYJP7"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F1kQkl1I2Gdt",
        "outputId": "3df871bf-cb5b-419e-8495-7c44294a03a5"
      },
      "source": [
        "with open(test_data_path, \"r\") as fp:\n",
        "    test_data = json.load(fp)\n",
        "test_data.keys()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['mfcc'])"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_blind = np.array(test_data[\"mfcc\"])\n",
        "X_blind =X_blind.reshape(X_blind.shape[0],-1,15,1)\n",
        "print(X_blind.shape)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zJQy4kuN3HJd",
        "outputId": "88a3c1da-efb9-46a8-b65e-338d412f5582"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(233, 302, 15, 1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_blind=model_200.predict(X_blind)\n",
        "# y_blind=model_20.predict(X_blind)\n",
        "y_blind=list(y_blind.reshape(-1))\n",
        "y_blind"
      ],
      "metadata": {
        "id": "U6tTVyJJ3R5C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df=pd.read_csv(test_metadata_path)\n",
        "df"
      ],
      "metadata": {
        "id": "cjPLnEd3ZeG1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iV1ebG6-Abnu"
      },
      "source": [
        "dicto={'File_name':list(df['File_name']),'Probability':y_blind}\n",
        "results=pd.DataFrame(dicto)\n",
        "results.to_csv( '/content/gdrive/MyDrive/Blind_Set_results_200.csv',index=False, header=False)\n",
        "# results.to_csv( '/content/gdrive/MyDrive/Blind_Set_results_20.csv',index=False, header=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ND56O_kVb9_"
      },
      "source": [
        "model_20.save('/content/gdrive/MyDrive/cough-correct-model_20.h5')\n",
        "# model_200.save('/content/gdrive/MyDrive/cough-correct-model.h5')\n",
        "\n",
        "# !tar -zcvf cough-it-model.tgz cough-it-model.h5"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yc3dmofx_vyV"
      },
      "source": [
        "!cp ./gdrive/MyDrive/COVID_cough/models/cough-it-model.h5 ./cough-it-model.h5 "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EjujcgyIt9dD"
      },
      "source": [
        "from keras.models import load_model\n",
        "\n",
        "model=load_model('/content/gdrive/MyDrive/COVID_cough/models/FULLcnn_mfcc15audiomentations.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I7MU-GMFUbFe"
      },
      "source": [
        "y_pred=model.predict(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O80XL6-F5dEr"
      },
      "source": [
        "from sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve\n",
        "\n",
        "fpr, tpr, thresholds = roc_curve(y_test, y_pred)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "zSMv8nY4I54t",
        "outputId": "6c371795-1f5c-4814-aa28-c4bdcd3ebd5e"
      },
      "source": [
        "sum_tests=0\n",
        "sum_preds=0\n",
        "\n",
        "for idx,item in enumerate(pred_folds_list):\n",
        "  sum_tests+=item['y_test']\n",
        "  sum_preds+=item['y_pred']\n",
        "\n",
        "avg_tests=sum_tests/len(pred_folds_list)\n",
        "avg_preds=sum_preds/len(pred_folds_list)\n",
        "\n",
        "fpr, tpr, thresholds = roc_curve(avg_tests, avg_preds)\n",
        "plt.plot(fpr, tpr, linestyle='--')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fdb96096e50>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 109
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAS7UlEQVR4nO3de4yddZ3H8ffXYhcE5LIdCaU3wKoMgkInFYIXVthNIQIJipRNUZBQYRdcL1FBN6g1cRc2KwbTVWoW0RJtgRDTYoWEikFIK8wIVDuIDBXs0AaKQI1cLLDf/eOcmunMaee0PTPPpe9X0uSc8/vNOZ+nM/30md/znOdEZiJJqr43FB1AktQZFrok1YSFLkk1YaFLUk1Y6JJUE3sV9cKTJk3KGTNmFPXyklRJfX19z2ZmV6uxwgp9xowZ9Pb2FvXyklRJEfHk9sZccpGkmrDQJakmLHRJqgkLXZJqwkKXpJoYtdAj4oaIeCYifrud8YiI6yJiICLWRMTxnY8pSRpNO3voNwJzdjB+GjCz+Wc+8J3djyVJ2lmjnoeemfdExIwdTDkL+GE2rsO7OiIOjIhDM3NjhzKOsHbDZhYs7x/x+BfmvJ1Z0w+m78nnuOaOR0eMX3VGN0dPPoB7H3uWb//8sRHj3zj7GI7s2o+7+p/me79cN2L82nPfzeQD92H5wxu4afXIU0G/M28WB+87kVt613Nr3+CI8RsvnM0+EyeweNUT3L5m5F/P0k+eCMCiex5n5SPPbDO29xsn8INPzAbgupWPcd/As9uMH/SmiXz3/FkAXH3H7/j1k89vM37oAXvzrbnHAfC15Wvp3/DnbcaP6NqX/zj7WACuvG0N6za9uM149+Q385Uzjgbg00seZOPmV7YZP376QXxxzjsAuGRxH8+/tGWb8ZPeOolPnTITgI/fcD+vvPr6NuOnHPUW5r//SADOvX4Vw33o2EM5/8QZvLzldS74/v0jxj8yawrn9EzluRe3cOlNfSPG550wnTPeNZkNL7zMZ5Y+NGL84vcdwandh/D4pr/wpdt+M2L88g/O5L0zJ/mz58/eiPFd+dnb+vfdaZ1YQz8MWD/k/mDzsREiYn5E9EZE76ZNmzrw0pKkraKdD7ho7qHfnpnvbDF2O/CfmXlv8/5K4IuZucO3gfb09OSuvFP03scaewfvnTlpp79WkqouIvoys6fVWCfe+v8UMHXI/SnNx8bE1l9XLXRJ2lYnllyWAR9rnu1yArB5LNfPJUmtjbqHHhE/Bk4GJkXEIPAV4I0AmfldYAVwOjAAvARcOFZhJUnb185ZLueNMp7Av3YskSRpl/hOUUmqicKuh76rvnH2MUVHkKRSqlyhH9m1X9ERJKmUKrfkclf/09zV/3TRMSSpdCq3h771bdGndh9ScBJJKpfK7aFLklqz0CWpJix0SaoJC12SaqJyB0WvPffdRUeQpFKqXKFPPnCfoiNIUilVbsll+cMbWP7whqJjSFLpVG4PfevHb53xrskFJ5GkcqncHrokqTULXZJqwkKXpJqw0CWpJip3UPQ782YVHUGSSqlyhX7wvhOLjiBJpVS5JZdbetdzS+/6omNIUulUrtBv7Rvk1r7BomNIUulUrtAlSa1Z6JJUExa6JNWEhS5JNVG50xZvvHB20REkqZQqV+j7TJxQdARJKqXKLbksXvUEi1c9UXAKSSqfyhX67Ws2cvuajUXHkKTSqVyhS5Jas9AlqSbaKvSImBMRj0bEQERc0WJ8WkTcHREPRsSaiDi981ElSTsyaqFHxARgIXAa0A2cFxHdw6b9O3BzZh4HzAX+p9NBJUk71s5pi7OBgcxcBxARS4CzgP4hcxJ4c/P2AcCGToYcauknTxyrp5akSmtnyeUwYOj1agebjw31VWBeRAwCK4DLWz1RRMyPiN6I6N20adMuxJUkbU+nDoqeB9yYmVOA04HFETHiuTNzUWb2ZGZPV1fXLr3QonseZ9E9j+9eWkmqoXYK/Slg6pD7U5qPDXURcDNAZq4C9gYmdSLgcCsfeYaVjzwzFk8tSZXWTqE/AMyMiMMjYiKNg57Lhs35I3AKQEQcRaPQXVORpHE0aqFn5mvAZcCdwCM0zmZZGxELIuLM5rTPARdHxMPAj4ELMjPHKrQkaaS2Ls6VmStoHOwc+thVQ273Ayd1NpokaWdU7mqLe7/Rqy1KUiuVK/QffMLroUtSK17LRZJqonKFft3Kx7hu5WNFx5Ck0qlcod838Cz3DTxbdAxJKp3KFbokqTULXZJqwkKXpJqo3GmLB71pYtERJKmUKlfo3z1/VtERJKmUXHKRpJqoXKFffcfvuPqO3xUdQ5JKp3JLLr9+8vmiI0hSKVVuD12S1JqFLkk1YaFLUk1Ubg390AP2LjqCJJVS5Qr9W3OPKzqCJJWSSy6SVBOVK/SvLV/L15avLTqGJJVO5ZZc+jf8uegIklRKldtDlyS1ZqFLUk1Y6JJUE5VbQz+ia9+iI0hSKVWu0P/j7GOLjiBJpeSSiyTVROUK/crb1nDlbWuKjiFJpVO5JZd1m14sOoIklVLl9tAlSa21VegRMSciHo2IgYi4YjtzPhoR/RGxNiJ+1NmYkqTRjLrkEhETgIXAPwKDwAMRsSwz+4fMmQlcCZyUmc9HxFvGKrAkqbV21tBnAwOZuQ4gIpYAZwH9Q+ZcDCzMzOcBMvOZTgfdqnvym8fqqSWp0top9MOA9UPuDwLvGTbnbQARcR8wAfhqZt4x/IkiYj4wH2DatGm7kpevnHH0Ln2dJNVdpw6K7gXMBE4GzgO+FxEHDp+UmYsysycze7q6ujr00pIkaK/QnwKmDrk/pfnYUIPAssx8NTP/APyeRsF33KeXPMinlzw4Fk8tSZXWTqE/AMyMiMMjYiIwF1g2bM5PaOydExGTaCzBrOtgzr/ZuPkVNm5+ZSyeWpIqbdRCz8zXgMuAO4FHgJszc21ELIiIM5vT7gT+FBH9wN3A5zPzT2MVWpI0UlvvFM3MFcCKYY9dNeR2Ap9t/pEkFcB3ikpSTVTuWi7HTz+o6AiSVEqVK/QvznlH0REkqZRccpGkmqhcoV+yuI9LFvcVHUOSSqdySy7Pv7Sl6AiSVEqV20OXJLVmoUtSTVjoklQTlVtDP+mtk4qOIEmlVLlC/9QpY3IRR0mqPJdcJKkmKlfoH7/hfj5+w/1Fx5Ck0qnckssrr75edARJKqXK7aFLklqz0CWpJix0SaqJyq2hn3LUW4qOIEmlVLlCn//+I4uOIEml5JKLJNVE5Qr93OtXce71q4qOIUmlU7lClyS1ZqFLUk1Y6JJUExa6JNVE5U5b/NCxhxYdQZJKqXKFfv6JM4qOIEmlVLkll5e3vM7LW7zioiQNV7lCv+D793PB970euiQNV7lClyS1ZqFLUk20VegRMSciHo2IgYi4YgfzPhwRGRE9nYsoSWrHqIUeEROAhcBpQDdwXkR0t5i3P/BvwK86HVKSNLp2TlucDQxk5jqAiFgCnAX0D5v3deBq4PMdTTjMR2ZNGcunl6TKamfJ5TBg/ZD7g83H/iYijgemZuZPd/REETE/InojonfTpk07HRbgnJ6pnNMzdZe+VpLqbLcPikbEG4BvAp8bbW5mLsrMnszs6erq2qXXe+7FLTz34pZd+lpJqrN2Cv0pYOgu8ZTmY1vtD7wT+EVEPAGcACwbqwOjl97Ux6U39Y3FU0tSpbVT6A8AMyPi8IiYCMwFlm0dzMzNmTkpM2dk5gxgNXBmZvaOSWJJUkujFnpmvgZcBtwJPALcnJlrI2JBRJw51gElSe1p6+JcmbkCWDHssau2M/fk3Y8lSdpZvlNUkmqicpfPnXfC9KIjSFIpVa7Qz3jX5KIjSFIpVW7JZcMLL7PhhZeLjiFJpVO5PfTPLH0IgKWfPLHgJJJULpXbQ5cktWahS1JNWOiSVBMWuiTVROUOil78viOKjiBJpVS5Qj+1+5CiI0hSKVVuyeXxTX/h8U1/KTqGJJVO5fbQv3TbbwDPQ5ek4Sq3hy5Jas1Cl6SasNAlqSYsdEmqicodFL38gzOLjiBJpVS5Qn/vzElFR5CkUqrcksvaDZtZu2Fz0TEkqXQqV+gLlvezYHl/0TEkqXQqV+iSpNYsdEmqCQtdkmrCQpekmqjcaYtfmPP2oiNIUilVrtBnTT+46AiSVEqVW3Lpe/I5+p58rugYklQ6lSv0a+54lGvueLToGJJUOpUrdElSa20VekTMiYhHI2IgIq5oMf7ZiOiPiDURsTIipnc+qiRpR0Yt9IiYACwETgO6gfMionvYtAeBnsw8FrgVuKbTQSVJO9bOHvpsYCAz12XmFmAJcNbQCZl5d2a+1Ly7GpjS2ZiSpNG0c9riYcD6IfcHgffsYP5FwM9aDUTEfGA+wLRp09qMuK2rzhj+y4EkCTp8HnpEzAN6gA+0Gs/MRcAigJ6entyV1zh68gG7nE+S6qydQn8KmDrk/pTmY9uIiFOBLwMfyMy/dibeSPc+9izgB11I0nDtFPoDwMyIOJxGkc8F/nnohIg4DrgemJOZz3Q85RDf/vljgIUuScONelA0M18DLgPuBB4Bbs7MtRGxICLObE77L2A/4JaIeCgilo1ZYklSS22toWfmCmDFsMeuGnL71A7nkiTtJN8pKkk1YaFLUk1U7vK53zj7mKIjSFIpVa7Qj+zar+gIklRKlVtyuav/ae7qf7roGJJUOpXbQ//eL9cBcGr3IQUnkaRyqdweuiSpNQtdkmrCQpekmrDQJakmKndQ9Npz3110BEkqpcoV+uQD9yk6giSVUuWWXJY/vIHlD28oOoYklU7l9tBvWv0kAGe8a3LBSSSpXCq3hy5Jas1Cl6SasNAlqSYsdEmqicodFP3OvFlFR5CkUqpcoR+878SiI0hSKVVuyeWW3vXc0ru+6BiSVDqVK/Rb+wa5tW+w6BiSVDqVK3RJUmsWuiTVhIUuSTVhoUtSTVTutMUbL5xddARJKqXKFfo+EycUHUGSSqlySy6LVz3B4lVPFJxCksqncoV++5qN3L5mY9ExJKl0KlfokqTW2ir0iJgTEY9GxEBEXNFi/O8iYmlz/FcRMaPTQSVJOzZqoUfEBGAhcBrQDZwXEd3Dpl0EPJ+ZbwWuBa7udFBJ0o61s4c+GxjIzHWZuQVYApw1bM5ZwA+at28FTomI6FxMSdJo2jlt8TBg6OUNB4H3bG9OZr4WEZuBvweeHTopIuYD8wGmTZu2S4GXfvLEXfo6Saq7cT0ompmLMrMnM3u6urrG86UlqfbaKfSngKlD7k9pPtZyTkTsBRwA/KkTASVJ7Wmn0B8AZkbE4RExEZgLLBs2Zxnw8ebtjwA/z8zsXExJ0mhGXUNvrolfBtwJTABuyMy1EbEA6M3MZcD/AosjYgB4jkbpS5LGUVvXcsnMFcCKYY9dNeT2K8A5nY0mSdoZvlNUkmrCQpekmrDQJakmLHRJqoko6uzCiNgEPLmLXz6JYe9C3QO4zXsGt3nPsDvbPD0zW74zs7BC3x0R0ZuZPUXnGE9u857Bbd4zjNU2u+QiSTVhoUtSTVS10BcVHaAAbvOewW3eM4zJNldyDV2SNFJV99AlScNY6JJUE6Uu9D3xw6nb2ObPRkR/RKyJiJURMb2InJ002jYPmffhiMiIqPwpbu1sc0R8tPm9XhsRPxrvjJ3Wxs/2tIi4OyIebP58n15Ezk6JiBsi4pmI+O12xiMirmv+fayJiON3+0Uzs5R/aFyq93HgCGAi8DDQPWzOvwDfbd6eCywtOvc4bPM/AG9q3r50T9jm5rz9gXuA1UBP0bnH4fs8E3gQOKh5/y1F5x6HbV4EXNq83Q08UXTu3dzm9wPHA7/dzvjpwM+AAE4AfrW7r1nmPfQ98cOpR93mzLw7M19q3l1N4xOkqqyd7zPA14GrgVfGM9wYaWebLwYWZubzAJn5zDhn7LR2tjmBNzdvHwBsGMd8HZeZ99D4fIjtOQv4YTasBg6MiEN35zXLXOitPpz6sO3NyczXgK0fTl1V7WzzUBfR+B++ykbd5uavolMz86fjGWwMtfN9fhvwtoi4LyJWR8SccUs3NtrZ5q8C8yJikMbnL1w+PtEKs7P/3kfV1gdcqHwiYh7QA3yg6CxjKSLeAHwTuKDgKONtLxrLLifT+C3snog4JjNfKDTV2DoPuDEz/zsiTqTxKWjvzMz/KzpYVZR5D31P/HDqdraZiDgV+DJwZmb+dZyyjZXRtnl/4J3ALyLiCRprjcsqfmC0ne/zILAsM1/NzD8Av6dR8FXVzjZfBNwMkJmrgL1pXMSqrtr6974zylzoe+KHU4+6zRFxHHA9jTKv+roqjLLNmbk5Mydl5ozMnEHjuMGZmdlbTNyOaOdn+yc09s6JiEk0lmDWjWfIDmtnm/8InAIQEUfRKPRN45pyfC0DPtY82+UEYHNmbtytZyz6SPAoR4lPp7Fn8jjw5eZjC2j8g4bGN/wWYAC4Hzii6MzjsM13AU8DDzX/LCs681hv87C5v6DiZ7m0+X0OGktN/cBvgLlFZx6Hbe4G7qNxBsxDwD8VnXk3t/fHwEbgVRq/cV0EXAJcMuR7vLD59/GbTvxc+9Z/SaqJMi+5SJJ2goUuSTVhoUtSTVjoklQTFrok1YSFLkk1YaFLUk38P93cji+kYKUvAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "C8zqyMzRAAew",
        "outputId": "a851bccf-8f2b-4ae7-ad25-dd234e721b47"
      },
      "source": [
        "fpr_list=[]\n",
        "tpr_list=[]\n",
        "\n",
        "for idx,item in enumerate(pred_folds_list):\n",
        "  \n",
        "  fpr, tpr, thresholds = roc_curve(item['y_test'], item['y_pred'])\n",
        "  fpr_list.append(fpr)\n",
        "  tpr_list.append(tpr)\n",
        "\n",
        "  plt.plot(fpr, tpr, linestyle='--')\n",
        "\n",
        "\n",
        "\n",
        "  \n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYK0lEQVR4nO3de3hU9Z3H8feXIGJQUAjKLRBqaW1WXS8ptNWutNVd0K10e7HQUhFRrK26VdeqaKGgpbVbr9VVaFHAO7i7GjTqU6k+WKuUUNAKigISA1guYmGVIrfv/jETnSRDZoDJnPM7+byeJ88zM79fZj6HhC8/vuc3Z8zdERGR8LWLOoCIiBSGCrqISEKooIuIJIQKuohIQqigi4gkRPuoXrisrMwrKiqienkRkSAtXLhwo7t3zzYWWUGvqKigtrY2qpcXEQmSmdXtaUwtFxGRhFBBFxFJCBV0EZGEUEEXEUkIFXQRkYTIWdDN7G4zW29mr+5h3MzsNjNbbmavmNkJhY8pIiK55LNCnw4MaWF8KDAg/TUWuHP/Y4mIyN7KuQ/d3eeZWUULU4YBMz11Hd6XzOxQM+vp7u8UKGNkZr8xm5qVNR/d//kXf06PTj146q2neHjZw5StGkC3+orU4K7tjPrhF+hZ/inun3Yr9a+XAtDpfad0a+oSxd/8xRl06dmH2Zf/hC0b+zR7vTEzLgDgoUvG88Hm3o3GjB2cO+MiAO7/wQS2fdCr0Xg7tjJ6xqUA3HfBRD7c1qPReIlt4ZzpVwAw87zr2LHj8MbPbxs5d/o1ANx9zs9wL2s83m4d594zHoBpo34JdGkcvv1axkybmBo/+2aw0sbf36Gec39zfXr8drADGucvrWP0nZPTzz+Fpko613HOryezvr6OOdc+1Wy8fdc6Rt08mdcXzueF2xY3Gz+wRz0jb7ieBb+r4ZX76puNl1b8lRETJ/DcrPtZ8cT7zcY7f/pdvjVuHE/+9i7WPm/Nxrsd/wFf+9FlPHrLTby7qFOz8V5fdIae931mT57MlmXdmo0fecbBDD7ruzw4YSJbV/VoNn7syHI+e9rp3HfltXz41/Jm4yddchxHnTiIGZeOY+emfs3Gv3r9EA4v78f0i8exa0vz8YbfvXsuHMfurU3GfQdjZqZ+9+4+/1p8e5PX962MmZn63Zs2ZgLsbPy7CZsZM+PHqe8fPQnffUSj0bb4u9fw511oheih9wYy/4asTj/WjJmNNbNaM6vdsGFDAV66ddWsrGHZpmV7HO9WX8FBm7um7uzaAQvvaTandKvTYXvMrzkf83gikh/L5wMu0iv0x9396CxjjwO/cPc/pO/PBa509xbfBlpVVeVxf6fo6KdGA3DPkOaFGuB/b/wzAP92+Qlwzxnpb3qi0Zy6750NQL97Z7ZSyv3z7SkvAvDwBZ+POImI5MPMFrp7VbaxQrz1fw2Q+X+wPunHimbJ82t440/rGj32ua8dSc8juzCvdgF//N836fS3v1P6/s6Pxo8Z2oGBw0fzzK2/ou7PhzR7zqqzujLq2FH8/f4/8uCDU5uND7nkBI4+pZy/P/MIdaeNge0fQIdO8PtUAe992620P+wwdm7cSPuysmbfH4UH5r/NY4vX0LNLR24ZfjwA89/axKD+XSNOJiKFUIiWSzVwdnq3y+eAzcXun7/xp3VsXN2875mp9P2de936GFw+mH4HZb0GDgADqo6g4rB066hDJ+jUfG6388+n87/+6169bmt5bPEalr6zpdFjo0+qYNhxWTtkIhKYnC0XM3sQGAyUAeuACcABAO5+l5kZcDupnTBbgdG52i1Q2JbLhvr/A6B7eeOV9uL1qRMUxx1+HHWnpVak/X63qCCvGSK1V0TCt18tF3cfkWPcgR/uY7aCaFrIG9z651uBVA+8+2WXFzOSiEjRRXb53EKY+Ug1axe9j7dzTviXfvzLySfz+vM3cMOK2QAcsXgbX1ri1N14PN0vu5zSoSMjTrz3GvreDavqqfNWMPe19Y3mdDyghBnnDgTgtrlv8sLyjY3GDyvtwF3fO5ET+h1WnNAiEomgC/raRe9z4N+68OGhm9n+Xrp1dPhnYNUBsGsHX1ri9F7vcHjLzxNnjy1ew/y3NhXkua4cclRBnkdE4imvbYutoRA99F9c8wAAV/3sO1nH475lMB/qe4tIptbetlhUS55fQ2mXA+l/bBklHTeza0v7j/eAA7QrgZMvhSO/VNRcDa0RgJu/fRy9Dj2IOS+v5b6Xmn+4yJ0jT6Rrpw7Mrq3nkYWrm41PHz2QgzqUcO+Lq1j6zhYqe3Zu7fgikgDBFfQ3/rSO3buc/seWMeT0o1g0b27jCbt3wXurADhi3NVFy9WwJbDQxbeyZ2dtKxSRvATXcml4d+ZhZ6X2Uw8uH1zIWPtMrRERKYZEtVwazFgyA7ZuYvDWv8OnhwLw3sOz2PL44x9Pal9C2fnn0+kLX9jr529ooYz/aiX/0KsLf3hzI7/+/ZvN5k3++jEc2f1gdu522rdrftEmEZFiCfsDLrasgT/e/vHdxx9n2+uvfzy+cxfb65v3qPOR7V2VLbnwlCPVGhGRSAW7Qs+mx8TUJTQP/ET/gjxfZc/O/EOv1KU6Tx5QxskD9nxNllMrj9jjmIhIMQRX0E8dXcm6utfgqlpKd++mjrUfXRCr6+jRHPLlj3e3ZO48Abj4ywM4eUAZS9ZuZtKcpc2e+8dDPs2J/bqysG6TdpeISHCCa7kc0rUjpZ1LKG1/EN0oaXHu3rZNMml3iYiEJrhdLtW/Sn2iyJn/ccEer0HeQDtPRCRpErXLZcOrBu7w6n/D15t/XFSmyV8/pkipRESiF1xBx52PPjOtS/PP5cx0ZPeDWz+PiEhMBNdDTzE4+hs5Zz2zdB3PLF2Xc56ISBIEWtBze2D+25w3s5bfPL8y6igiIkWR2ILesF1RO1VEpK0Iroc+9IoT8547qH9XvjOobyumERGJj+AKeq+js+7WERFp84Ir6P9z9XgAvv7zSS3Ou/nbxxUjjohIbARX0N9bm19PvNehB7VyEhGReEnsSdE5L69lzstro44hIlI0wa3Q89Xw0W9f/cdeEScRESmOxK7QRUTaGhV0EZGECK7l8s1fnBF1BBGRWAquoHfp2fIFuURE2qrgCvrsy38CwLduvK7FeXeOzP8dpSIiSRBcQd+yMb8VetdOHVo5iYhIvCT2pOjs2npm19ZHHUNEpGgSW9AfWbiaRxaujjqGiEjR5FXQzWyImS0zs+VmdlWW8b5m9qyZLTKzV8zs9MJHFRGRluQs6GZWAtwBDAUqgRFmVtlk2rXALHc/HhgO/Fehg4qISMvyOSk6EFju7isBzOwhYBiwNGOOA53Tt7sArXYRlTEzLmitpxYRCVo+LZfeQObZxdXpxzL9FBhpZquBGuDibE9kZmPNrNbMajds2LAPcfM3qH/XVn1+EZG4KdS2xRHAdHe/0cw+D9xrZke7++7MSe4+FZgKUFVV5fvyQg9dkroe+vDbsl8P/d4XVwFw4eBP0qPLmn15CRGRIOVT0NcA5Rn3+6QfyzQGGALg7i+aWUegDFhfiJCZPtjc8vXQH3/lHQC+9/kKffyciLQp+bRcFgADzKy/mXUgddKzusmct4GvAJjZZ4COQOv2VEREpJGcBd3ddwIXAU8Dr5HazbLEzCaZ2ZnpaZcD55vZy8CDwDnuvk8tFRER2Td59dDdvYbUyc7Mx8Zn3F4KnFTYaCIisjeCu5aLsSPqCCIisWRRdUaqqqq8trY2ktcWEQmVmS1096psY4m6lsvUeSuYOm9F1DFERCIRXEG//wcTuP8HE7KOzX1tPXNfK/hOSRGRIATXQ9/2Qa+oI4iIxFJwK3QREclOBV1EJCGCLugPzH+bb095kdvmvglAxwNK6HhAScSpRESiEVwPvR1bP7r92OI1LH1nCyd9sgyAGecOjCqWiEjkgivoo2dc2uh+Zc/OXPKVARGlERGJj6BbLiIi8rHgVuj3XTARgJFTJnBYaYeI04iIxEdwBf3DbT0+un3X906MMImISLwE23J5YP7b3PDU61HHEBGJjWAL+mOL13Dnc7pui4hIg2ALOuiDoEVEMgXXQy+xLVFHEBGJpeAK+jnTrwDgySkvRpxERCRegivoDXp26Rh1BBGRWAmuoM887zoAbvntTyJOIiISL8EV9B07Do86gohILAW7y2XinCVMnLMk6hgiIrER3Aq9wdK12u0iIpIp2BW6iIg0FtwKvX3JpqgjiIjEUnAr9FF3X82ou69m/lsq7CIimYJboTcYMbCcY3ofGnUMEZHYCK6g333OzwD4+fRrIk4iIhIvwRV097KoI4iIxFJwPXQREckur4JuZkPMbJmZLTezq/Yw5ywzW2pmS8zsgcLGFBGRXHK2XMysBLgDOA1YDSwws2p3X5oxZwBwNXCSu79nZnp/vohIkeWzQh8ILHf3le6+HXgIGNZkzvnAHe7+HoC7ry9szI9Zu3VYu3Wt9fQiIsHK56Rob6A+4/5qYFCTOZ8CMLMXgBLgp+7+VNMnMrOxwFiAvn377ktezr1n/D59n4hI0hXqpGh7YAAwGBgB/MbMmm0Sd/ep7l7l7lXdu3cv0EuLiAjkV9DXAOUZ9/ukH8u0Gqh29x3u/hbwBqkCX3DTRv2SaaN+2RpPLSIStHwK+gJggJn1N7MOwHCgusmcR0mtzjGzMlItmJUFzJmhS/pLREQy5Szo7r4TuAh4GngNmOXuS8xskpmdmZ72NPCumS0FngWucPd3Wyu0iIg0l9c7Rd29Bqhp8tj4jNsOXJb+EhGRCOidoiIiCRHctVxovzbqBCIisRRcQR8zbWLUEUREYkktFxGRhAiuoE87+2amnX1z1DFERGInuJYLVhp1AhGRWApuhS4iItmpoIuIJIQKuohIQgTXQ7cO9bkniYi0QcEV9HN/c33UEUREYkktFxGRhAiuoE87+3amnX171DFERGInuJYLdkDUCUREYim4FbqIiGSngi4ikhAq6CIiCRFcD71daV3UEUREYim4gj76zslRRxARiSW1XEREEiK4gj5t1BSmjZoSdQwRkdgJrqCLiEh2KugiIgmhgi4ikhAq6CIiCRHctsWSztqHLiKSTXAF/Zxfax+6iEg2wbVc1tfXsb5eq3QRkaaCK+hzrn2KOdc+FXUMEZHYCa6gi4hIdiroIiIJkVdBN7MhZrbMzJab2VUtzPuGmbmZVRUuooiI5CNnQTezEuAOYChQCYwws8os8w4B/h2YX+iQIiKSWz4r9IHAcndf6e7bgYeAYVnmXQfcAGwrYL5m2neto31X7XIREWkqn33ovYH6jPurgUGZE8zsBKDc3Z8wsyv29ERmNhYYC9C3b9+9TwuMuln70EVEstnvk6Jm1g64Cbg811x3n+ruVe5e1b179316vdcXzuf1herqiIg0lc8KfQ1QnnG/T/qxBocARwPPmRlAD6DazM5099pCBW3wwm2LAThqxqAcM0VE2pZ8VugLgAFm1t/MOgDDgeqGQXff7O5l7l7h7hXAS0CrFHMREdmznAXd3XcCFwFPA68Bs9x9iZlNMrMzWzugiIjkJ6+Lc7l7DVDT5LHxe5g7eP9jiYjI3tI7RUVEEiK4y+ce2KM+9yQRkTYouII+8obro44gIhJLwbVcFvyuhgW/q8k9UUSkjQluhf7KfamWy2dPiziIiEjMBLdCFxGR7FTQRUQSQgVdRCQhVNBFRBIiuJOipRV/jTqCiEgsBVfQR0ycEHUEEZFYCq7l8tys+3lu1v1RxxARiZ3gCvqKJ95nxRPvRx1DRCR2givoIiKSnQq6iEhCqKCLiCSECrqISEIEt22x86ffjTqCiEgsBVfQvzVuXNQRRERiKbiWy5O/vYsnf3tX1DFERGInuIK+9nlj7fMWdQwRkdgJrqCLiEh2KugiIgmhgi4ikhAq6CIiCRHctsVux38QdQQRkVgKrqB/7UeXRR1BRCSWgmu5PHrLTTx6y01RxxARiZ3gVujvLuoUdQQRkVgKboUuIiLZ5VXQzWyImS0zs+VmdlWW8cvMbKmZvWJmc82sX+GjiohIS3IWdDMrAe4AhgKVwAgzq2wybRFQ5e7HAo8Avyx0UBERaVk+K/SBwHJ3X+nu24GHgGGZE9z9WXffmr77EtCnsDFFRCSXfE6K9gbqM+6vBga1MH8M8GS2ATMbC4wF6Nu3b54RG+v1Rd+n7xMRSbqC7nIxs5FAFXBKtnF3nwpMBaiqqtqnyjz0vO/vcz4RkSTLp6CvAcoz7vdJP9aImZ0KXAOc4u4fFiZec7MnTwb0QRciIk3lU9AXAAPMrD+pQj4c+E7mBDM7HpgCDHH39QVPmWHLsm6t+fQiIsHKeVLU3XcCFwFPA68Bs9x9iZlNMrMz09P+EzgYmG1mi82sutUSi4hIVnn10N29Bqhp8tj4jNunFjiXiIjsJb1TVEQkIVTQRUQSIriLcx15xsFRRxARiaXgCvrgs74bdQQRkVgKruXy4ISJPDhhYtQxRERiJ7gV+tZVPaKOICISS8Gt0EVEJDsVdBGRhFBBFxFJCBV0EZGECO6k6LEjy3NPEhFpg4Ir6J897fSoI4iIxFJwLZf7rryW+668NuoYIiKxE9wK/cO/quUiIpJNcCt0ERHJTgVdRCQhVNBFRBJCBV1EJCGCOyl60iXHRR1BRCSWgivoR504KOoIIiKxFFzLZcal45hx6bioY4iIxE5wK/Sdm/pFHUFEJJaCW6GLiEh2KugiIgmhgi4ikhAq6CIiCRHcSdGvXj8k6ggiIrEUXEE/vFy7XEREsgmu5TL94nFMv1j70EVEmgpuhb5ri1boIiLZBLdCFxGR7PIq6GY2xMyWmdlyM7sqy/iBZvZweny+mVUUOqiIiLQsZ0E3sxLgDmAoUAmMMLPKJtPGAO+5+yeBm4EbCh1URERals8KfSCw3N1Xuvt24CFgWJM5w4AZ6duPAF8xMytcTBERySWfk6K9gfqM+6uBptew/WiOu+80s81AN2Bj5iQzGwuMBejbt+8+BR4z44J9+j4RkaQr6klRd5/q7lXuXtW9e/divrSISOLlU9DXAOUZ9/ukH8s6x8zaA12AdwsRUERE8pNPQV8ADDCz/mbWARgOVDeZUw2MSt/+JvB7d/fCxRQRkVxy9tDTPfGLgKeBEuBud19iZpOAWnevBqYB95rZcmATqaIvIiJFlNc7Rd29Bqhp8tj4jNvbgG8VNpqIiOwNvVNURCQhVNBFRBJCBV1EJCFU0EVEEsKi2l1oZhuAun389jKavAu1DdAxtw065rZhf465n7tnfWdmZAV9f5hZrbtXRZ2jmHTMbYOOuW1orWNWy0VEJCFU0EVEEiLUgj416gAR0DG3DTrmtqFVjjnIHrqIiDQX6gpdRESaUEEXEUmIWBf0tvjh1Hkc82VmttTMXjGzuWbWL4qchZTrmDPmfcPM3MyC3+KWzzGb2Vnpn/USM3ug2BkLLY/f7b5m9qyZLUr/fp8eRc5CMbO7zWy9mb26h3Ezs9vSfx6vmNkJ+/2i7h7LL1KX6l0BfALoALwMVDaZ8wPgrvTt4cDDUecuwjF/CShN376wLRxzet4hwDzgJaAq6txF+DkPABYBh6XvHx517iIc81TgwvTtSmBV1Ln385j/CTgBeHUP46cDTwIGfA6Yv7+vGecVelv8cOqcx+zuz7r71vTdl0h9glTI8vk5A1wH3ABsK2a4VpLPMZ8P3OHu7wG4+/oiZyy0fI7Zgc7p212AtUXMV3DuPo/U50PsyTBgpqe8BBxqZj335zXjXNCzfTh17z3NcfedQMOHU4cqn2PONIbUv/Ahy3nM6f+Klrv7E8UM1ory+Tl/CviUmb1gZi+Z2ZCipWsd+RzzT4GRZraa1OcvXFycaJHZ27/vOeX1ARcSP2Y2EqgCTok6S2sys3bATcA5EUcptvak2i6DSf0vbJ6ZHePuf4s0VesaAUx39xvN7POkPgXtaHffHXWwUMR5hd4WP5w6n2PGzE4FrgHOdPcPi5StteQ65kOAo4HnzGwVqV5jdeAnRvP5Oa8Gqt19h7u/BbxBqsCHKp9jHgPMAnD3F4GOpC5ilVR5/X3fG3Eu6G3xw6lzHrOZHQ9MIVXMQ++rQo5jdvfN7l7m7hXuXkHqvMGZ7l4bTdyCyOd3+1FSq3PMrIxUC2ZlMUMWWD7H/DbwFQAz+wypgr6hqCmLqxo4O73b5XPAZnd/Z7+eMeozwTnOEp9OamWyArgm/dgkUn+hIfUDnw0sB/4EfCLqzEU45meAdcDi9Fd11Jlb+5ibzH2OwHe55PlzNlKtpqXAX4DhUWcuwjFXAi+Q2gGzGPjnqDPv5/E+CLwD7CD1P64xwPeB72f8jO9I/3n8pRC/13rrv4hIQsS55SIiIntBBV1EJCFU0EVEEkIFXUQkIVTQRUQSQgVdRCQhVNBFRBLi/wEQ9S8NyOKKnwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "7i71ezaK5oih",
        "outputId": "a464fd58-5c0c-466f-e822-938526a1c470"
      },
      "source": [
        "plt.plot(fpr, tpr, linestyle='--')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fdcb982dd90>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAS7UlEQVR4nO3de4yddZ3H8ffXYhcE5LIdCaU3wKoMgkInFYIXVthNIQIJipRNUZBQYRdcL1FBN6g1cRc2KwbTVWoW0RJtgRDTYoWEikFIK8wIVDuIDBXs0AaKQI1cLLDf/eOcmunMaee0PTPPpe9X0uSc8/vNOZ+nM/30md/znOdEZiJJqr43FB1AktQZFrok1YSFLkk1YaFLUk1Y6JJUE3sV9cKTJk3KGTNmFPXyklRJfX19z2ZmV6uxwgp9xowZ9Pb2FvXyklRJEfHk9sZccpGkmrDQJakmLHRJqgkLXZJqwkKXpJoYtdAj4oaIeCYifrud8YiI6yJiICLWRMTxnY8pSRpNO3voNwJzdjB+GjCz+Wc+8J3djyVJ2lmjnoeemfdExIwdTDkL+GE2rsO7OiIOjIhDM3NjhzKOsHbDZhYs7x/x+BfmvJ1Z0w+m78nnuOaOR0eMX3VGN0dPPoB7H3uWb//8sRHj3zj7GI7s2o+7+p/me79cN2L82nPfzeQD92H5wxu4afXIU0G/M28WB+87kVt613Nr3+CI8RsvnM0+EyeweNUT3L5m5F/P0k+eCMCiex5n5SPPbDO29xsn8INPzAbgupWPcd/As9uMH/SmiXz3/FkAXH3H7/j1k89vM37oAXvzrbnHAfC15Wvp3/DnbcaP6NqX/zj7WACuvG0N6za9uM149+Q385Uzjgbg00seZOPmV7YZP376QXxxzjsAuGRxH8+/tGWb8ZPeOolPnTITgI/fcD+vvPr6NuOnHPUW5r//SADOvX4Vw33o2EM5/8QZvLzldS74/v0jxj8yawrn9EzluRe3cOlNfSPG550wnTPeNZkNL7zMZ5Y+NGL84vcdwandh/D4pr/wpdt+M2L88g/O5L0zJ/mz58/eiPFd+dnb+vfdaZ1YQz8MWD/k/mDzsREiYn5E9EZE76ZNmzrw0pKkraKdD7ho7qHfnpnvbDF2O/CfmXlv8/5K4IuZucO3gfb09OSuvFP03scaewfvnTlpp79WkqouIvoys6fVWCfe+v8UMHXI/SnNx8bE1l9XLXRJ2lYnllyWAR9rnu1yArB5LNfPJUmtjbqHHhE/Bk4GJkXEIPAV4I0AmfldYAVwOjAAvARcOFZhJUnb185ZLueNMp7Av3YskSRpl/hOUUmqicKuh76rvnH2MUVHkKRSqlyhH9m1X9ERJKmUKrfkclf/09zV/3TRMSSpdCq3h771bdGndh9ScBJJKpfK7aFLklqz0CWpJix0SaoJC12SaqJyB0WvPffdRUeQpFKqXKFPPnCfoiNIUilVbsll+cMbWP7whqJjSFLpVG4PfevHb53xrskFJ5GkcqncHrokqTULXZJqwkKXpJqw0CWpJip3UPQ782YVHUGSSqlyhX7wvhOLjiBJpVS5JZdbetdzS+/6omNIUulUrtBv7Rvk1r7BomNIUulUrtAlSa1Z6JJUExa6JNWEhS5JNVG50xZvvHB20REkqZQqV+j7TJxQdARJKqXKLbksXvUEi1c9UXAKSSqfyhX67Ws2cvuajUXHkKTSqVyhS5Jas9AlqSbaKvSImBMRj0bEQERc0WJ8WkTcHREPRsSaiDi981ElSTsyaqFHxARgIXAa0A2cFxHdw6b9O3BzZh4HzAX+p9NBJUk71s5pi7OBgcxcBxARS4CzgP4hcxJ4c/P2AcCGToYcauknTxyrp5akSmtnyeUwYOj1agebjw31VWBeRAwCK4DLWz1RRMyPiN6I6N20adMuxJUkbU+nDoqeB9yYmVOA04HFETHiuTNzUWb2ZGZPV1fXLr3QonseZ9E9j+9eWkmqoXYK/Slg6pD7U5qPDXURcDNAZq4C9gYmdSLgcCsfeYaVjzwzFk8tSZXWTqE/AMyMiMMjYiKNg57Lhs35I3AKQEQcRaPQXVORpHE0aqFn5mvAZcCdwCM0zmZZGxELIuLM5rTPARdHxMPAj4ELMjPHKrQkaaS2Ls6VmStoHOwc+thVQ273Ayd1NpokaWdU7mqLe7/Rqy1KUiuVK/QffMLroUtSK17LRZJqonKFft3Kx7hu5WNFx5Ck0qlcod838Cz3DTxbdAxJKp3KFbokqTULXZJqwkKXpJqo3GmLB71pYtERJKmUKlfo3z1/VtERJKmUXHKRpJqoXKFffcfvuPqO3xUdQ5JKp3JLLr9+8vmiI0hSKVVuD12S1JqFLkk1YaFLUk1Ubg390AP2LjqCJJVS5Qr9W3OPKzqCJJWSSy6SVBOVK/SvLV/L15avLTqGJJVO5ZZc+jf8uegIklRKldtDlyS1ZqFLUk1Y6JJUE5VbQz+ia9+iI0hSKVWu0P/j7GOLjiBJpeSSiyTVROUK/crb1nDlbWuKjiFJpVO5JZd1m14sOoIklVLl9tAlSa21VegRMSciHo2IgYi4YjtzPhoR/RGxNiJ+1NmYkqTRjLrkEhETgIXAPwKDwAMRsSwz+4fMmQlcCZyUmc9HxFvGKrAkqbV21tBnAwOZuQ4gIpYAZwH9Q+ZcDCzMzOcBMvOZTgfdqnvym8fqqSWp0top9MOA9UPuDwLvGTbnbQARcR8wAfhqZt4x/IkiYj4wH2DatGm7kpevnHH0Ln2dJNVdpw6K7gXMBE4GzgO+FxEHDp+UmYsysycze7q6ujr00pIkaK/QnwKmDrk/pfnYUIPAssx8NTP/APyeRsF33KeXPMinlzw4Fk8tSZXWTqE/AMyMiMMjYiIwF1g2bM5PaOydExGTaCzBrOtgzr/ZuPkVNm5+ZSyeWpIqbdRCz8zXgMuAO4FHgJszc21ELIiIM5vT7gT+FBH9wN3A5zPzT2MVWpI0UlvvFM3MFcCKYY9dNeR2Ap9t/pEkFcB3ikpSTVTuWi7HTz+o6AiSVEqVK/QvznlH0REkqZRccpGkmqhcoV+yuI9LFvcVHUOSSqdySy7Pv7Sl6AiSVEqV20OXJLVmoUtSTVjoklQTlVtDP+mtk4qOIEmlVLlC/9QpY3IRR0mqPJdcJKkmKlfoH7/hfj5+w/1Fx5Ck0qnckssrr75edARJKqXK7aFLklqz0CWpJix0SaqJyq2hn3LUW4qOIEmlVLlCn//+I4uOIEml5JKLJNVE5Qr93OtXce71q4qOIUmlU7lClyS1ZqFLUk1Y6JJUExa6JNVE5U5b/NCxhxYdQZJKqXKFfv6JM4qOIEmlVLkll5e3vM7LW7zioiQNV7lCv+D793PB970euiQNV7lClyS1ZqFLUk20VegRMSciHo2IgYi4YgfzPhwRGRE9nYsoSWrHqIUeEROAhcBpQDdwXkR0t5i3P/BvwK86HVKSNLp2TlucDQxk5jqAiFgCnAX0D5v3deBq4PMdTTjMR2ZNGcunl6TKamfJ5TBg/ZD7g83H/iYijgemZuZPd/REETE/InojonfTpk07HRbgnJ6pnNMzdZe+VpLqbLcPikbEG4BvAp8bbW5mLsrMnszs6erq2qXXe+7FLTz34pZd+lpJqrN2Cv0pYOgu8ZTmY1vtD7wT+EVEPAGcACwbqwOjl97Ux6U39Y3FU0tSpbVT6A8AMyPi8IiYCMwFlm0dzMzNmTkpM2dk5gxgNXBmZvaOSWJJUkujFnpmvgZcBtwJPALcnJlrI2JBRJw51gElSe1p6+JcmbkCWDHssau2M/fk3Y8lSdpZvlNUkmqicpfPnXfC9KIjSFIpVa7Qz3jX5KIjSFIpVW7JZcMLL7PhhZeLjiFJpVO5PfTPLH0IgKWfPLHgJJJULpXbQ5cktWahS1JNWOiSVBMWuiTVROUOil78viOKjiBJpVS5Qj+1+5CiI0hSKVVuyeXxTX/h8U1/KTqGJJVO5fbQv3TbbwDPQ5ek4Sq3hy5Jas1Cl6SasNAlqSYsdEmqicodFL38gzOLjiBJpVS5Qn/vzElFR5CkUqrcksvaDZtZu2Fz0TEkqXQqV+gLlvezYHl/0TEkqXQqV+iSpNYsdEmqCQtdkmrCQpekmqjcaYtfmPP2oiNIUilVrtBnTT+46AiSVEqVW3Lpe/I5+p58rugYklQ6lSv0a+54lGvueLToGJJUOpUrdElSa20VekTMiYhHI2IgIq5oMf7ZiOiPiDURsTIipnc+qiRpR0Yt9IiYACwETgO6gfMionvYtAeBnsw8FrgVuKbTQSVJO9bOHvpsYCAz12XmFmAJcNbQCZl5d2a+1Ly7GpjS2ZiSpNG0c9riYcD6IfcHgffsYP5FwM9aDUTEfGA+wLRp09qMuK2rzhj+y4EkCTp8HnpEzAN6gA+0Gs/MRcAigJ6entyV1zh68gG7nE+S6qydQn8KmDrk/pTmY9uIiFOBLwMfyMy/dibeSPc+9izgB11I0nDtFPoDwMyIOJxGkc8F/nnohIg4DrgemJOZz3Q85RDf/vljgIUuScONelA0M18DLgPuBB4Bbs7MtRGxICLObE77L2A/4JaIeCgilo1ZYklSS22toWfmCmDFsMeuGnL71A7nkiTtJN8pKkk1YaFLUk1U7vK53zj7mKIjSFIpVa7Qj+zar+gIklRKlVtyuav/ae7qf7roGJJUOpXbQ//eL9cBcGr3IQUnkaRyqdweuiSpNQtdkmrCQpekmrDQJakmKndQ9Npz3110BEkqpcoV+uQD9yk6giSVUuWWXJY/vIHlD28oOoYklU7l9tBvWv0kAGe8a3LBSSSpXCq3hy5Jas1Cl6SasNAlqSYsdEmqicodFP3OvFlFR5CkUqpcoR+878SiI0hSKVVuyeWW3vXc0ru+6BiSVDqVK/Rb+wa5tW+w6BiSVDqVK3RJUmsWuiTVhIUuSTVhoUtSTVTutMUbL5xddARJKqXKFfo+EycUHUGSSqlySy6LVz3B4lVPFJxCksqncoV++5qN3L5mY9ExJKl0KlfokqTW2ir0iJgTEY9GxEBEXNFi/O8iYmlz/FcRMaPTQSVJOzZqoUfEBGAhcBrQDZwXEd3Dpl0EPJ+ZbwWuBa7udFBJ0o61s4c+GxjIzHWZuQVYApw1bM5ZwA+at28FTomI6FxMSdJo2jlt8TBg6OUNB4H3bG9OZr4WEZuBvweeHTopIuYD8wGmTZu2S4GXfvLEXfo6Saq7cT0ompmLMrMnM3u6urrG86UlqfbaKfSngKlD7k9pPtZyTkTsBRwA/KkTASVJ7Wmn0B8AZkbE4RExEZgLLBs2Zxnw8ebtjwA/z8zsXExJ0mhGXUNvrolfBtwJTABuyMy1EbEA6M3MZcD/AosjYgB4jkbpS5LGUVvXcsnMFcCKYY9dNeT2K8A5nY0mSdoZvlNUkmrCQpekmrDQJakmLHRJqoko6uzCiNgEPLmLXz6JYe9C3QO4zXsGt3nPsDvbPD0zW74zs7BC3x0R0ZuZPUXnGE9u857Bbd4zjNU2u+QiSTVhoUtSTVS10BcVHaAAbvOewW3eM4zJNldyDV2SNFJV99AlScNY6JJUE6Uu9D3xw6nb2ObPRkR/RKyJiJURMb2InJ002jYPmffhiMiIqPwpbu1sc0R8tPm9XhsRPxrvjJ3Wxs/2tIi4OyIebP58n15Ezk6JiBsi4pmI+O12xiMirmv+fayJiON3+0Uzs5R/aFyq93HgCGAi8DDQPWzOvwDfbd6eCywtOvc4bPM/AG9q3r50T9jm5rz9gXuA1UBP0bnH4fs8E3gQOKh5/y1F5x6HbV4EXNq83Q08UXTu3dzm9wPHA7/dzvjpwM+AAE4AfrW7r1nmPfQ98cOpR93mzLw7M19q3l1N4xOkqqyd7zPA14GrgVfGM9wYaWebLwYWZubzAJn5zDhn7LR2tjmBNzdvHwBsGMd8HZeZ99D4fIjtOQv4YTasBg6MiEN35zXLXOitPpz6sO3NyczXgK0fTl1V7WzzUBfR+B++ykbd5uavolMz86fjGWwMtfN9fhvwtoi4LyJWR8SccUs3NtrZ5q8C8yJikMbnL1w+PtEKs7P/3kfV1gdcqHwiYh7QA3yg6CxjKSLeAHwTuKDgKONtLxrLLifT+C3snog4JjNfKDTV2DoPuDEz/zsiTqTxKWjvzMz/KzpYVZR5D31P/HDqdraZiDgV+DJwZmb+dZyyjZXRtnl/4J3ALyLiCRprjcsqfmC0ne/zILAsM1/NzD8Av6dR8FXVzjZfBNwMkJmrgL1pXMSqrtr6974zylzoe+KHU4+6zRFxHHA9jTKv+roqjLLNmbk5Mydl5ozMnEHjuMGZmdlbTNyOaOdn+yc09s6JiEk0lmDWjWfIDmtnm/8InAIQEUfRKPRN45pyfC0DPtY82+UEYHNmbtytZyz6SPAoR4lPp7Fn8jjw5eZjC2j8g4bGN/wWYAC4Hzii6MzjsM13AU8DDzX/LCs681hv87C5v6DiZ7m0+X0OGktN/cBvgLlFZx6Hbe4G7qNxBsxDwD8VnXk3t/fHwEbgVRq/cV0EXAJcMuR7vLD59/GbTvxc+9Z/SaqJMi+5SJJ2goUuSTVhoUtSTVjoklQTFrok1YSFLkk1YaFLUk38P93cji+kYKUvAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SAy-SCNkU0ea",
        "outputId": "4157cacc-d1f3-4204-93b7-f99093b6e5c8"
      },
      "source": [
        "\n",
        "\n",
        "auc = roc_auc_score(y_test, y_pred)\n",
        "print('AUC: %.3f' % auc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "AUC: 1.000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BZ1W-NjkVfGP"
      },
      "source": [
        "#model.save('cnn_auc888.h5',save_format='h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_06wFDwp9YE3"
      },
      "source": [
        "#Lmodel=load_model('cnn_auc888.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R-vQk5adXxL5",
        "outputId": "7615dd7b-c82c-4b5e-9d6e-04fefdc3f370"
      },
      "source": [
        "y_test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0,\n",
              "       0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0,\n",
              "       0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1,\n",
              "       1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0,\n",
              "       0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1,\n",
              "       0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1cRuwLF8ZUkE"
      },
      "source": [
        "y_preds=[]\n",
        "\n",
        "for i in y_pred:\n",
        "  if i >=0.4:\n",
        "    y_preds.append(1)\n",
        "  else:\n",
        "    y_preds.append(0)\n",
        "\n",
        "y_preds=np.array(y_preds)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rpgvWlB2aX-f",
        "outputId": "f796958b-13f8-4d8b-c17a-d523edee88ee"
      },
      "source": [
        "y_preds"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0,\n",
              "       0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0,\n",
              "       0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1,\n",
              "       1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0,\n",
              "       0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1,\n",
              "       0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S2n49mVEafx3",
        "outputId": "29c213da-4b75-49d4-cbec-84a21f42037d"
      },
      "source": [
        "confusion_matrix(y_test, y_preds)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[184,   9],\n",
              "       [ 12,  51]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O1Y3HOuRwH2e"
      },
      "source": [
        "model.save('/content/gdrive/MyDrive/COVID_cough/models/mfcc15audiomentations.h5',save_format='h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fWFZx0Q0VRFY"
      },
      "source": [
        "Blind set eval"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p0IxRyNdVNgH"
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kvQC7SmoVSwR"
      },
      "source": [
        "from keras.models import load_model\n",
        "\n",
        "model=load_model(r'/content/gdrive/MyDrive/COVID_cough/models/mfcc15audiomentations.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2A1FfWuNVWo1"
      },
      "source": [
        "df=pd.read_csv('/content/gdrive/MyDrive/COVID_cough/test_metadata.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "id": "AxaKVpqwVcmk",
        "outputId": "f2687c8d-90df-4749-e597-080e673414f7"
      },
      "source": [
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>File_name</th>\n",
              "      <th>Covid_status</th>\n",
              "      <th>Gender</th>\n",
              "      <th>Nationality</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ehmkjtFk_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>m</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>UCxrHdoP_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>m</td>\n",
              "      <td>I</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>afJveyWf_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>f</td>\n",
              "      <td>I</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ckgXCsSh_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>m</td>\n",
              "      <td>I</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>KprOlhNx_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>m</td>\n",
              "      <td>I</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>228</th>\n",
              "      <td>WxtUSymX_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>f</td>\n",
              "      <td>I</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>229</th>\n",
              "      <td>VYFvgWMm_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>f</td>\n",
              "      <td>I</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>230</th>\n",
              "      <td>coWKLPUF_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>f</td>\n",
              "      <td>I</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>231</th>\n",
              "      <td>XvOqdxQJ_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>m</td>\n",
              "      <td>I</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232</th>\n",
              "      <td>fyfYKFsi_cough</td>\n",
              "      <td>?</td>\n",
              "      <td>m</td>\n",
              "      <td>I</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>233 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          File_name Covid_status Gender Nationality\n",
              "0    ehmkjtFk_cough            ?      m           O\n",
              "1    UCxrHdoP_cough            ?      m           I\n",
              "2    afJveyWf_cough            ?      f           I\n",
              "3    ckgXCsSh_cough            ?      m           I\n",
              "4    KprOlhNx_cough            ?      m           I\n",
              "..              ...          ...    ...         ...\n",
              "228  WxtUSymX_cough            ?      f           I\n",
              "229  VYFvgWMm_cough            ?      f           I\n",
              "230  coWKLPUF_cough            ?      f           I\n",
              "231  XvOqdxQJ_cough            ?      m           I\n",
              "232  fyfYKFsi_cough            ?      m           I\n",
              "\n",
              "[233 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5JFJePUvZkQX"
      },
      "source": [
        "data_path = '/content/gdrive/MyDrive/COVID_cough/data/Evaluations/mfcc15_blind_set.json'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nrSSmjCuZpXt"
      },
      "source": [
        "with open(data_path, \"r\") as fp:\n",
        "        data = json.load(fp)\n",
        "        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CpDb9XziZqts",
        "outputId": "6c6f53bf-c295-46ff-e1e0-8530c9c28cad"
      },
      "source": [
        "X = np.array(data[\"mfcc\"])\n",
        "X=X.reshape(X.shape[0],-1,15,1)\n",
        "X.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(233, 302, 15, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gL8bkgOrZtrH"
      },
      "source": [
        "y_pred_probs=model.predict(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S3AW8ko2ZwW8"
      },
      "source": [
        "y=list(y_pred_probs.reshape(-1))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z9pizTaqZxa5"
      },
      "source": [
        "dicto={'File_name':list(df['File_name']),'Probability':y}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w08VZrpbZyu8"
      },
      "source": [
        "results=pd.DataFrame(dicto)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KvLVa4NIZzrE"
      },
      "source": [
        "results.to_csv('15mfcc_audiomentation_Blind_Set_results.csv',index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z_jo8SyEZ3Aq"
      },
      "source": [
        "x=pd.read_csv('15mfcc_audiomentation_Blind_Set_results.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EvWM3bFkZ5xA",
        "outputId": "29c67403-1a2f-40ee-f328-c6ce50c2322d"
      },
      "source": [
        "type(x['Probability'][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "numpy.float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YdSkXM16aBnv"
      },
      "source": [
        "Val Folds eval"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WnSkjI8DaGcB"
      },
      "source": [
        "df=pd.read_csv('/content/gdrive/MyDrive/COVID_cough/Val_set.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "id": "O-bqzNzpaHc0",
        "outputId": "e6afa4e1-8efe-4599-ea85-6dedcb8e5585"
      },
      "source": [
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>file_name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PQFgfbUS_cough</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>dpITIaDY_cough</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>IzXaepIB_cough</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ZVKSsFiW_cough</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>LFqjhpiv_cough</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1085</th>\n",
              "      <td>tWjZnGra_cough</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1086</th>\n",
              "      <td>dAlFPhsp_cough</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1087</th>\n",
              "      <td>gjLXOetR_cough</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1088</th>\n",
              "      <td>WJAIhgpc_cough</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1089</th>\n",
              "      <td>FcWxqeAl_cough</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1090 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           file_name\n",
              "0     PQFgfbUS_cough\n",
              "1     dpITIaDY_cough\n",
              "2     IzXaepIB_cough\n",
              "3     ZVKSsFiW_cough\n",
              "4     LFqjhpiv_cough\n",
              "...              ...\n",
              "1085  tWjZnGra_cough\n",
              "1086  dAlFPhsp_cough\n",
              "1087  gjLXOetR_cough\n",
              "1088  WJAIhgpc_cough\n",
              "1089  FcWxqeAl_cough\n",
              "\n",
              "[1090 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OQmgz1XiaIZH"
      },
      "source": [
        "data_path = '/content/gdrive/MyDrive/COVID_cough/data/Evaluations/mfcc15_val_folds.json'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5r4SEi9iaL9i"
      },
      "source": [
        "with open(data_path, \"r\") as fp:\n",
        "        data = json.load(fp)\n",
        "        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5LgB96t6aN-J",
        "outputId": "d43411aa-c508-4168-9086-d59f946a04fa"
      },
      "source": [
        "X = np.array(data[\"mfcc\"])\n",
        "X=X.reshape(X.shape[0],-1,15,1)\n",
        "X.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1090, 302, 15, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q2UAchlEaRkV"
      },
      "source": [
        "y_pred_probs=model.predict(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sW4FPmD-aU5j"
      },
      "source": [
        "y=list(y_pred_probs.reshape(-1))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "grK9DqhJaV2-"
      },
      "source": [
        "dicto={'File_name':list(df['file_name']),'Probability':y}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SMtpJPgxaW25"
      },
      "source": [
        "results=pd.DataFrame(dicto)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LppJ9ewxaX1o"
      },
      "source": [
        "results.to_csv('15mfcc_audiomentations_val_set_results.csv',index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WqY80HG7aZD0"
      },
      "source": [
        "x=pd.read_csv('15mfcc_audiomentations_val_set_results.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zPjDx8AZakk_",
        "outputId": "3e21ebc6-525d-4476-b2a1-5d3cc2b29a87"
      },
      "source": [
        "type(x['Probability'][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "numpy.float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B53ueBbsal9t"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}