train-from-file-generic-clean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:46.644188Z",
     "start_time": "2020-05-01T14:26:44.729363Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext nb_black\n",
    "import os\n",
    "from pathlib import Path\n",
    "from tensorflow.keras.applications import (\n",
    "    MobileNetV2,\n",
    "    NASNetLarge,\n",
    "    NASNetMobile,\n",
    "    InceptionV3,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:46.650730Z",
     "start_time": "2020-05-01T14:26:46.646043Z"
    }
   },
   "outputs": [],
   "source": [
    "# !pip install efficientnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:47.015942Z",
     "start_time": "2020-05-01T14:26:46.653273Z"
    }
   },
   "outputs": [],
   "source": [
    "from efficientnet.tfkeras import (\n",
    "    EfficientNetB0,\n",
    "    EfficientNetB4,\n",
    "    EfficientNetB2,\n",
    "    EfficientNetB3,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:47.043380Z",
     "start_time": "2020-05-01T14:26:47.017764Z"
    }
   },
   "outputs": [],
   "source": [
    "# dataset_name = \"santander-customer-satisfaction\"\n",
    "# IMAGE_SIZE = 640\n",
    "# dataset_name = \"springleaf-marketing-response\"\n",
    "# IMAGE_SIZE = 1408\n",
    "# dataset_name = \"segment\"\n",
    "# IMAGE_SIZE = 160\n",
    "dataset_name = \"rl\"\n",
    "# IMAGE_SIZE = 160\n",
    "# dataset_name = \"open-payments\"\n",
    "# IMAGE_SIZE = 96\n",
    "dataset_name = \"bank-marketing\"\n",
    "IMAGE_SIZE = 160\n",
    "# dataset_name = \"springleaf-marketing-response\"\n",
    "# IMAGE_SIZE = 1408\n",
    "# dataset_name = \"bnp-cardif\"\n",
    "# IMAGE_SIZE = 384\n",
    "# dataset_name = \"albert\"\n",
    "# IMAGE_SIZE = 288\n",
    "# dataset_name = \"titanic\"\n",
    "# IMAGE_SIZE = 128\n",
    "# dataset_name = \"cat-in-the-dat-ii\"\n",
    "# IMAGE_SIZE = 96\n",
    "# dataset_name = \"give-me-some-credit\"\n",
    "# IMAGE_SIZE = 128\n",
    "# dataset_name = \"census-income\"\n",
    "# IMAGE_SIZE = 224\n",
    "\n",
    "# IMAGE_SIZE = 160\n",
    "\n",
    "DATASET_FOLDER = Path(os.getcwd()) / f\"data/{dataset_name}\"\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "PREFETCH = 50\n",
    "\n",
    "\n",
    "ONE_CHANNEL = True\n",
    "NB_CHANNEL = 1 if ONE_CHANNEL else 3\n",
    "\n",
    "# Target size for model to use\n",
    "TARGET_SIZE = IMAGE_SIZE\n",
    "PRETRAINED_MODEL = MobileNetV2  # EfficientNetB0  # EfficientNetB4  # MobileNetV2  # EfficientNetB2  # EfficientNetB0  # MobileNetV2  # NASNetMobile\n",
    "FROM_LAYER_RETRAIN = 0  # 119\n",
    "OUTPUT_LAYER = (\n",
    "    None  # -4  # None  # Only for efficient Net, since include tops does not work None\n",
    ")\n",
    "\n",
    "# Training params for only new layers\n",
    "epochs_1 = 50\n",
    "patience_1 = 10\n",
    "\n",
    "# Training params for refit\n",
    "epochs_2 = 200\n",
    "patience_2 = 20\n",
    "\n",
    "# Model name => image size, last_block retrain\n",
    "# NASNetMobile => 224,\n",
    "# NASNetLarge => 331,\n",
    "# MobileNetV2 => 96, ... 160..... 224, last layer => 128 (3 blocks), 137(2 blocks), 146(1 block)\n",
    "# InceptionV3 => 299,\n",
    "# Xception => 299,"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-17T13:04:39.963383Z",
     "start_time": "2020-04-17T13:04:39.955914Z"
    }
   },
   "source": [
    "## Import + utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:47.971257Z",
     "start_time": "2020-05-01T14:26:47.045854Z"
    }
   },
   "outputs": [],
   "source": [
    "import gc\n",
    "import json\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import (\n",
    "    ZeroPadding2D,\n",
    "    Dense,\n",
    "    GlobalAveragePooling2D,\n",
    "    AveragePooling2D,\n",
    "    Input,\n",
    "    Dropout,\n",
    ")\n",
    "\n",
    "from tensorflow_addons.optimizers import RectifiedAdam, Lookahead\n",
    "from tensorflow_addons.activations import mish\n",
    "from concurrent.futures import ProcessPoolExecutor as PoolExecutor\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import imshow\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:47.988033Z",
     "start_time": "2020-05-01T14:26:47.973203Z"
    }
   },
   "outputs": [],
   "source": [
    "classnames = None\n",
    "with (DATASET_FOLDER / \"prep_data\" / str(IMAGE_SIZE) / \"classnames.json\").open() as fp:\n",
    "    classnames = np.array(json.load(fp))\n",
    "OUTPUT_DIM = len(classnames)\n",
    "LOSS = \"binary_crossentropy\" if OUTPUT_DIM == 2 else \"categorical_crossentropy\"\n",
    "METRIC = \"AUC\" if OUTPUT_DIM == 2 else \"accuracy\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:48.060426Z",
     "start_time": "2020-05-01T14:26:47.989966Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_metric(history, metric):\n",
    "    # Plot training & validation loss values\n",
    "    plt.plot(history.history[metric])\n",
    "    plt.plot(history.history[f\"val_{metric}\"])\n",
    "    plt.title(f\"Model {metric}\")\n",
    "    plt.ylabel(f\"{metric}\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.legend([\"Train\", \"Test\"], loc=\"upper left\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-17T13:09:05.896414Z",
     "start_time": "2020-04-17T13:09:05.860854Z"
    }
   },
   "source": [
    "## Preparing datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:48.069749Z",
     "start_time": "2020-05-01T14:26:48.063594Z"
    }
   },
   "outputs": [],
   "source": [
    "# https://www.tensorflow.org/tutorials/load_data/images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:48.116957Z",
     "start_time": "2020-05-01T14:26:48.074303Z"
    }
   },
   "outputs": [],
   "source": [
    "file_list = None\n",
    "with (DATASET_FOLDER / \"prep_data\" / str(IMAGE_SIZE) / \"file_list.json\").open() as fp:\n",
    "    file_list = json.load(fp)\n",
    "file_list[\"train\"][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:48.157865Z",
     "start_time": "2020-05-01T14:26:48.118843Z"
    }
   },
   "outputs": [],
   "source": [
    "def build_process_path(classnames):\n",
    "    def process_path(file_path):\n",
    "        label = tf.strings.split(file_path, os.path.sep)[-2]\n",
    "        label = label == classnames\n",
    "        # load the raw data from the file as a string\n",
    "        img = tf.io.read_file(file_path)\n",
    "        img = tf.io.decode_compressed(img, \"GZIP\")\n",
    "        # convert the compressed string to a 3D uint8 tensor\n",
    "        img = tf.image.decode_jpeg(img, channels=NB_CHANNEL)\n",
    "        return img, label\n",
    "\n",
    "    return process_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:48.246359Z",
     "start_time": "2020-05-01T14:26:48.160506Z"
    }
   },
   "outputs": [],
   "source": [
    "classnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:48.327696Z",
     "start_time": "2020-05-01T14:26:48.249644Z"
    }
   },
   "outputs": [],
   "source": [
    "process_path = build_process_path(classnames)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "    output_shapes=(tf.TensorShape((None, None, None)), tf.TensorShape((1, )))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:48.420124Z",
     "start_time": "2020-05-01T14:26:48.330731Z"
    }
   },
   "outputs": [],
   "source": [
    "AUTOTUNE = tf.data.experimental.AUTOTUNE\n",
    "\n",
    "\n",
    "def build_dataset(file_list, process_path, *, repeat, batch_size, prefetch):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(file_list)\n",
    "    if repeat:\n",
    "        dataset = dataset.repeat()\n",
    "\n",
    "    dataset = dataset.map(process_path, num_parallel_calls=AUTOTUNE)\n",
    "    if batch_size is not None:\n",
    "        dataset = dataset.batch(batch_size)\n",
    "\n",
    "    if prefetch is not None:\n",
    "        dataset = dataset.prefetch(prefetch)\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:50.609746Z",
     "start_time": "2020-05-01T14:26:48.422307Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_train = build_dataset(\n",
    "    file_list[\"train\"],\n",
    "    process_path,\n",
    "    repeat=True,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    prefetch=PREFETCH,\n",
    ")\n",
    "dataset_valid = build_dataset(\n",
    "    file_list[\"valid\"],\n",
    "    process_path,\n",
    "    repeat=True,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    prefetch=PREFETCH,\n",
    ")\n",
    "dataset_test = build_dataset(\n",
    "    file_list[\"test\"], process_path, repeat=False, batch_size=BATCH_SIZE, prefetch=None\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:51.328604Z",
     "start_time": "2020-05-01T14:26:50.611327Z"
    }
   },
   "outputs": [],
   "source": [
    "for image, label in dataset_train.take(1):\n",
    "    print(\"Image shape: \", image.numpy().shape)\n",
    "    print(\"Label: \", label.numpy().shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Images Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:26:51.341965Z",
     "start_time": "2020-05-01T14:26:51.330844Z"
    }
   },
   "outputs": [],
   "source": [
    "def show_image(image, image_size, one_channel=False):\n",
    "    if one_channel:\n",
    "        imshow(image.reshape(IMAGE_SIZE, IMAGE_SIZE))\n",
    "    else:\n",
    "        imshow(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:01.907522Z",
     "start_time": "2020-05-01T14:26:51.344213Z"
    }
   },
   "outputs": [],
   "source": [
    "for image, label in dataset_train.take(1):\n",
    "    print(\"Label: \", label[0].numpy())\n",
    "    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:02.138731Z",
     "start_time": "2020-05-01T14:27:01.909745Z"
    }
   },
   "outputs": [],
   "source": [
    "for image, label in dataset_valid.take(1):\n",
    "    print(\"Label: \", label[0].numpy())\n",
    "    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training model (transfer learning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:02.151297Z",
     "start_time": "2020-05-01T14:27:02.140618Z"
    }
   },
   "outputs": [],
   "source": [
    "steps_per_epoch = np.ceil(len(file_list[\"train\"]) / BATCH_SIZE)\n",
    "steps_per_epoch_val = np.ceil(len(file_list[\"valid\"]) / BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model, using existing one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:02.185133Z",
     "start_time": "2020-05-01T14:27:02.153112Z"
    }
   },
   "outputs": [],
   "source": [
    "activation = mish\n",
    "optimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:04.430538Z",
     "start_time": "2020-05-01T14:27:02.187246Z"
    }
   },
   "outputs": [],
   "source": [
    "nb_channel = 1 if ONE_CHANNEL else 3\n",
    "\n",
    "# Base model have precise input shape\n",
    "# In order to match, I do Pooling + Padding (can be asymetric) to match it\n",
    "\n",
    "POOLING = int(np.ceil(IMAGE_SIZE / TARGET_SIZE))\n",
    "PADDING = np.floor(TARGET_SIZE - np.floor(IMAGE_SIZE / POOLING))\n",
    "PADDING_ASYM = int(PADDING % 2)\n",
    "\n",
    "PADDING = int(np.floor(PADDING / 2))\n",
    "\n",
    "\n",
    "# Let's divide the size to be smaller than input of base_model\n",
    "# inputs = AveragePooling2D(pool_size=(POOLING, POOLING))(inputs)\n",
    "# Let's pad to adapt to precise shape\n",
    "# inputs = ZeroPadding2D(\n",
    "#    padding=((PADDING, PADDING + PADDING_ASYM), (PADDING, PADDING + PADDING_ASYM),)\n",
    "# )(inputs)\n",
    "\n",
    "# This is the input of our new model\n",
    "inputs = Input(shape=(IMAGE_SIZE, IMAGE_SIZE, nb_channel))\n",
    "input_layer = inputs\n",
    "\n",
    "\n",
    "# Now, we can use PRE TRAINED model\n",
    "base_model = PRETRAINED_MODEL(\n",
    "    input_tensor=inputs,\n",
    "    input_shape=(TARGET_SIZE, TARGET_SIZE, nb_channel),\n",
    "    weights=None,  # \"imagenet\",\n",
    "    # weights=\"noisy-student\",\n",
    "    # weights=None,\n",
    "    include_top=False,\n",
    ")\n",
    "\n",
    "x = (\n",
    "    base_model.output\n",
    "    if OUTPUT_LAYER is None or OUTPUT_LAYER >= -1\n",
    "    else base_model.layers[OUTPUT_LAYER].output\n",
    ")\n",
    "# add a global spatial average pooling layer\n",
    "x = GlobalAveragePooling2D()(x)\n",
    "# let's add a fully-connected layer\n",
    "# x = Dense(1024, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "# x = Dense(512, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "\n",
    "x = Dense(1024, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "# x = Dropout(0.2)(x)\n",
    "x = Dense(512, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "# x = Dropout(0.2)(x)\n",
    "x = Dense(128, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "# x = Dropout(0.2)(x)\n",
    "\n",
    "\n",
    "# x = Dense(256, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "# x = Dropout(0.2)(x)\n",
    "# x = Dense(128, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "# x = Dropout(0.2)(x)\n",
    "# and a logistic layer -- let's say we have 200 classes\n",
    "predictions = Dense(OUTPUT_DIM, activation=\"softmax\")(x)\n",
    "\n",
    "# this is the model we will train\n",
    "model = Model(inputs=input_layer, outputs=predictions)\n",
    "\n",
    "# first: train only the top layers (which were randomly initialized)\n",
    "# i.e. freeze all base_model layers\n",
    "for layer in base_model.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "# compile the model (should be done *after* setting layers to non-trainable)\n",
    "es = EarlyStopping(\n",
    "    monitor=\"val_loss\",\n",
    "    verbose=1,\n",
    "    mode=\"min\",\n",
    "    patience=patience_1,\n",
    "    restore_best_weights=True,\n",
    ")\n",
    "# We need to recompile the model for these modifications to take effect\n",
    "es.set_model(model)\n",
    "model.compile(optimizer=optimizer, loss=LOSS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:04.475710Z",
     "start_time": "2020-05-01T14:27:04.431985Z"
    }
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:04.552244Z",
     "start_time": "2020-05-01T14:27:04.480719Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# let's visualize layer names and layer indices to see how many layers\n",
    "# we should freeze:\n",
    "for i, layer in enumerate(model.layers):\n",
    "    print(i, layer.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit new layers"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# train the model on the new data for a few epochs\n",
    "history_1 = model.fit(\n",
    "    dataset_train,\n",
    "    callbacks=[es],\n",
    "    epochs=epochs_1,\n",
    "    steps_per_epoch=steps_per_epoch,\n",
    "    validation_data=dataset_valid,\n",
    "    validation_steps=steps_per_epoch_val,\n",
    ")\n",
    "# at this point, the top layers are well trained and we can start fine-tuning\n",
    "# convolutional layers from inception V3. We will freeze the bottom N layers\n",
    "# and train the remaining top layers."
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "plot_metric(history_1, \"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:04.949548Z",
     "start_time": "2020-05-01T14:27:04.554720Z"
    }
   },
   "outputs": [],
   "source": [
    "truth_test = []\n",
    "for _, labels in dataset_test:\n",
    "    truth_test.append(np.argmax(labels, axis=1))\n",
    "truth_test = np.hstack(truth_test)\n",
    "truth_test\n",
    "\n",
    "truth_valid = []\n",
    "for i, (_, labels) in enumerate(dataset_valid):\n",
    "    truth_valid.append(np.argmax(labels, axis=1))\n",
    "    if i >= steps_per_epoch_val - 1:\n",
    "        break\n",
    "truth_valid = np.hstack(truth_valid)\n",
    "truth_valid.shape"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "preds_valid = model.predict(dataset_valid, steps=steps_per_epoch_val)\n",
    "preds_valid.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(\n",
    "        f\"Accuracy valid: {accuracy_score(truth_valid, np.argmax(preds_valid, axis=1))}\"\n",
    "    )\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC valid: {roc_auc_score(truth_valid, preds_valid[:, 1])}\")\n",
    "preds_test = model.predict(dataset_test)\n",
    "preds_test.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(f\"Accuracy test: {accuracy_score(truth_test, np.argmax(preds_test, axis=1))}\")\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC test: {roc_auc_score(truth_test, preds_test[:, 1])}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unfreeze and fit more/all layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:04.971298Z",
     "start_time": "2020-05-01T14:27:04.951113Z"
    }
   },
   "outputs": [],
   "source": [
    "# Let's freeze the whole model\n",
    "for layer in model.layers:\n",
    "    layer.trainable = False\n",
    "# Now, we can unfreeze everything, or only some block\n",
    "for layer in model.layers[FROM_LAYER_RETRAIN:]:\n",
    "    layer.trainable = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:27:05.056779Z",
     "start_time": "2020-05-01T14:27:04.972778Z"
    }
   },
   "outputs": [],
   "source": [
    "# Let's build an optimizer\n",
    "optimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)\n",
    "es = EarlyStopping(\n",
    "    monitor=\"val_loss\",\n",
    "    verbose=1,\n",
    "    mode=\"min\",\n",
    "    patience=patience_2,\n",
    "    restore_best_weights=True,\n",
    ")\n",
    "# We need to recompile the model for these modifications to take effect\n",
    "es.set_model(model)\n",
    "model.compile(optimizer=optimizer, loss=LOSS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:37:39.033859Z",
     "start_time": "2020-05-01T14:27:05.058385Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# we train our model again (this time fine-tuning the top 2 inception blocks\n",
    "# alongside the top Dense layers\n",
    "history_2 = model.fit(\n",
    "    dataset_train,\n",
    "    callbacks=[es],\n",
    "    epochs=epochs_2,\n",
    "    steps_per_epoch=steps_per_epoch,\n",
    "    validation_data=dataset_valid,\n",
    "    validation_steps=steps_per_epoch_val,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:37:39.036673Z",
     "start_time": "2020-05-01T14:26:44.913Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_metric(history_2, \"loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-17T13:16:20.349217Z",
     "start_time": "2020-04-17T13:16:16.612Z"
    }
   },
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:37:39.038277Z",
     "start_time": "2020-05-01T14:26:44.916Z"
    }
   },
   "outputs": [],
   "source": [
    "preds_valid = model.predict(dataset_valid, steps=steps_per_epoch_val)\n",
    "preds_valid.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(\n",
    "        f\"Accuracy valid: {accuracy_score(truth_valid, np.argmax(preds_valid, axis=1))}\"\n",
    "    )\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC valid: {roc_auc_score(truth_valid, preds_valid[:, 1])}\")\n",
    "preds_test = model.predict(dataset_test)\n",
    "preds_test.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(f\"Accuracy test: {accuracy_score(truth_test, np.argmax(preds_test, axis=1))}\")\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC test: {roc_auc_score(truth_test, preds_test[:, 1])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:37:39.039899Z",
     "start_time": "2020-05-01T14:26:44.919Z"
    }
   },
   "outputs": [],
   "source": [
    "# Open payments 13min, 0.9338321957865205 (batch 256)\n",
    "# cat-in-the-dat-ii 0.7643929251726093 1h10 (batch 256)\n",
    "# RL ROC AUC test: 0.9430279072306678 mobile net patience 20\n",
    "# BNP cardif (image size : 96) ROC AUC test: 0.7168011573597879 1h16\n",
    "# give-me-some-credit ROC AUC test: 0.8591709381962962 2h4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:37:39.041508Z",
     "start_time": "2020-05-01T14:26:44.922Z"
    }
   },
   "outputs": [],
   "source": [
    "# Open payments 1h10\n",
    "# ROC AUC valid: 0.9432323040315754\n",
    "# ROC AUC test: 0.9345809216081824"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:37:39.043044Z",
     "start_time": "2020-05-01T14:26:44.926Z"
    }
   },
   "outputs": [],
   "source": [
    "# EfficientNetB0\n",
    "# EfficientNetB2 2h => ROC AUC valid: 0.9357130861335854 ROC AUC test: 0.9330937133279599"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:37:39.044843Z",
     "start_time": "2020-05-01T14:26:44.928Z"
    }
   },
   "outputs": [],
   "source": [
    "# \"santander-customer-satisfaction\"\n",
    "# 0.8164865598696714 => target size 96, whole re train\n",
    "# 0.8141824599511267 => target size 224, whole re train\n",
    "# ROC AUC valid: 0.8268400760249797 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# give me some credit\n",
    "# 0.8462134942186483 => target size 160, whole train (batch 128, 2layers 1024, 128)\n",
    "\n",
    "# ROC AUC valid: 0.8348917439829162\n",
    "# RL\n",
    "# ROC AUC valid: 0.892118469133795 => 160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.9233534348199217 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9465346534653467 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# Open payment\n",
    "# ROC AUC valid: 0.9186458210299415 => 96 =>160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.8778868370932499 => 96 => 96, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.8917381493730192 => 96 => 224, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.9090374872044725 => 96 => 96, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC valid: 0.886417393797122 => 96 => 160, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC valid: 0.9045819676568436 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9045819676568436 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9406275221953189 => 96 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9399110034154216 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "# ROC AUC valid: 0.8811034128677376 => 96 => 96, whole, batch 128, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# Bank marketing\n",
    "# ROC AUC valid: 0.7970734141661526 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# Albert\n",
    "# ROC AUC valid: 0.7500980687987842 => 288 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# bnp-cardif\n",
    "# ROC AUC valid: 0.7206667869818926 => 384 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-01T14:37:39.046445Z",
     "start_time": "2020-05-01T14:26:44.931Z"
    }
   },
   "outputs": [],
   "source": [
    "# \"santander-customer-satisfaction\"\n",
    "# 0.833417731838137 => target size 96, whole train\n",
    "# 0.8170226029745679 => target size 224, whole train\n",
    "# ROC AUC test: 0.8348549041045967 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# give me some credit\n",
    "# 0.8453497574694486 => target size 160, whole train (batch 128, 2layers 1024, 128)\n",
    "# ROC AUC test: 0.8447058873195916\n",
    "\n",
    "# RL\n",
    "# ROC AUC test: 0.9051288159651395 => 160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.9128674518211912 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "# ROC AUC test: 0.9456874816987527 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "# ROC AUC test: 0.9399110034154216 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# Open payment\n",
    "# ROC AUC test: 0.916907759155486 => 96 =>160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.8931101859362467 => 96 => 96, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.8938387451368033 => 96 => 224, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.9067445823812874 => 96 => 96, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC test: 0.8954549557710788 => 96 => 160, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC test: 0.9062895529860363 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout\n",
    "# ROC AUC test: 0.9403517762951931 => 96 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# Bank marketing\n",
    "# ROC AUC valid: 0.7959000291791145 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# Albert\n",
    "# ROC AUC test: 0.7487660412524685 => 288 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# bnp-cardif\n",
    "# ROC AUC test: 0.725934546476426 => 384 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "notify_time": "5"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}