train-from-csv-generic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:42.494953Z",
     "start_time": "2020-04-29T16:51:42.392663Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext nb_black\n",
    "import os\n",
    "from pathlib import Path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:43.793137Z",
     "start_time": "2020-04-29T16:51:42.495931Z"
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import io\n",
    "from requests import get\n",
    "from pathlib import Path\n",
    "import shutil\n",
    "import gzip\n",
    "import gc\n",
    "\n",
    "from itertools import repeat\n",
    "\n",
    "from concurrent.futures import ProcessPoolExecutor as PoolExecutor\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "from PIL import Image, ImageDraw, ImageFont\n",
    "from matplotlib.pyplot import imshow\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import gc\n",
    "import json\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import (\n",
    "    ZeroPadding2D,\n",
    "    Dense,\n",
    "    GlobalAveragePooling2D,\n",
    "    AveragePooling2D,\n",
    "    Input,\n",
    "    Dropout,\n",
    ")\n",
    "\n",
    "from tensorflow_addons.optimizers import RectifiedAdam, Lookahead\n",
    "from tensorflow_addons.activations import mish\n",
    "from concurrent.futures import ProcessPoolExecutor as PoolExecutor\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import imshow\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:43.802149Z",
     "start_time": "2020-04-29T16:51:43.794443Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"open-payments\"\n",
    "target = \"status\"\n",
    "\n",
    "panda_kwargs = {}\n",
    "\n",
    "DATASET_FOLDER = Path(os.getcwd()) / f\"data/{dataset_name}\"\n",
    "DATASET_FILENAME = \"train_bench.csv\"\n",
    "dataset_path = DATASET_FOLDER / DATASET_FILENAME\n",
    "FONT_FOLDER = Path(os.getcwd())\n",
    "\n",
    "TAILORED_COLUMN = \"Set\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:43.812059Z",
     "start_time": "2020-04-29T16:51:43.803047Z"
    }
   },
   "outputs": [],
   "source": [
    "IMAGE_SIZE = 96\n",
    "CUT_LENGTH = None\n",
    "ONE_CHANNEL = True\n",
    "NB_CHANNEL = 1 if ONE_CHANNEL else 3\n",
    "patience = 5\n",
    "epochs = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:43.820817Z",
     "start_time": "2020-04-29T16:51:43.812870Z"
    }
   },
   "outputs": [],
   "source": [
    "columns = pd.read_csv(dataset_path, **panda_kwargs, nrows=1).columns.tolist()\n",
    "columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:43.887575Z",
     "start_time": "2020-04-29T16:51:43.821623Z"
    }
   },
   "outputs": [],
   "source": [
    "target_values = pd.read_csv(\n",
    "    dataset_path, **panda_kwargs, usecols=[target]\n",
    ").values.reshape(-1)\n",
    "CLASSNAMES = np.unique(target_values).astype(\"str\")\n",
    "NB_LINES = target_values.shape[0]\n",
    "del target_values\n",
    "print(NB_LINES)\n",
    "print(CLASSNAMES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:43.893081Z",
     "start_time": "2020-04-29T16:51:43.888425Z"
    }
   },
   "outputs": [],
   "source": [
    "OUTPUT_DIM = len(CLASSNAMES)\n",
    "LOSS = \"binary_crossentropy\" if OUTPUT_DIM == 2 else \"categorical_crossentropy\"\n",
    "METRIC = \"AUC\" if OUTPUT_DIM == 2 else \"accuracy\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:43.944159Z",
     "start_time": "2020-04-29T16:51:43.894478Z"
    }
   },
   "outputs": [],
   "source": [
    "split = None\n",
    "if TAILORED_COLUMN not in columns:\n",
    "    split = np.random.choice(\n",
    "        [\"train\", \"valid\", \"test\"], p=[0.8, 0.1, 0.1], size=(NB_LINES,)\n",
    "    )\n",
    "else:\n",
    "    split = pd.read_csv(\n",
    "        dataset_path, **panda_kwargs, usecols=[TAILORED_COLUMN]\n",
    "    ).values.reshape(-1)\n",
    "\n",
    "\n",
    "train_indices = np.argwhere(split == \"train\").reshape(-1)\n",
    "np.random.shuffle(train_indices)\n",
    "valid_indices = np.argwhere(split == \"valid\").reshape(-1)\n",
    "test_indices = np.argwhere(split == \"test\").reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:43.949790Z",
     "start_time": "2020-04-29T16:51:43.945374Z"
    }
   },
   "outputs": [],
   "source": [
    "used_columns = list(set(columns) - set([TAILORED_COLUMN, target]))\n",
    "used_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.012537Z",
     "start_time": "2020-04-29T16:51:43.950546Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(dataset_path, **panda_kwargs, usecols=used_columns + [target])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.432691Z",
     "start_time": "2020-04-29T16:51:44.013552Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train = df[used_columns].values[train_indices].astype(\"str\")\n",
    "Y_train = df[[target]].values[train_indices].astype(\"str\")\n",
    "\n",
    "X_valid = df[used_columns].values[train_indices].astype(\"str\")\n",
    "Y_valid = df[[target]].values[train_indices].astype(\"str\")\n",
    "\n",
    "X_test = df[used_columns].values[train_indices].astype(\"str\")\n",
    "Y_test = df[[target]].values[train_indices].astype(\"str\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.436739Z",
     "start_time": "2020-04-29T16:51:44.433704Z"
    }
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "PREFETCH = 100000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.446587Z",
     "start_time": "2020-04-29T16:51:44.437536Z"
    }
   },
   "outputs": [],
   "source": [
    "steps_per_epoch = np.ceil(X_train.shape[0] / BATCH_SIZE)\n",
    "steps_per_epoch_val = np.ceil(X_valid.shape[0] / BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.451557Z",
     "start_time": "2020-04-29T16:51:44.447476Z"
    }
   },
   "outputs": [],
   "source": [
    "del df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T12:10:43.056592Z",
     "start_time": "2020-04-29T12:10:43.054673Z"
    }
   },
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.462772Z",
     "start_time": "2020-04-29T16:51:44.452420Z"
    }
   },
   "outputs": [],
   "source": [
    "def download(url, out, force=False, verify=True):\n",
    "    out.parent.mkdir(parents=True, exist_ok=True)\n",
    "    if force:\n",
    "        print(f\"Removing file at {str(out)}\")\n",
    "        out.unlink()\n",
    "\n",
    "    if out.exists():\n",
    "        print(\"File already exists.\")\n",
    "        return\n",
    "    print(f\"Downloading {url} at {str(out)} ...\")\n",
    "    # open in binary mode\n",
    "    with out.open(mode=\"wb\") as file:\n",
    "        # get request\n",
    "        response = get(url, verify=verify)\n",
    "        for chunk in response.iter_content(100000):\n",
    "            # write to file\n",
    "            file.write(chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.468253Z",
     "start_time": "2020-04-29T16:51:44.463645Z"
    }
   },
   "outputs": [],
   "source": [
    "font_url = \"https://ff.static.1001fonts.net/r/o/roboto-condensed.regular.ttf\"\n",
    "\n",
    "dataset_path = DATASET_FOLDER / DATASET_FILENAME\n",
    "out_font = FONT_FOLDER / f\"RobotoCondensed-Regular.ttf\"\n",
    "\n",
    "download(font_url, out_font)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.489303Z",
     "start_time": "2020-04-29T16:51:44.469068Z"
    }
   },
   "outputs": [],
   "source": [
    "def word_to_square_image(text, size, cut_length=None, one_channel=False):\n",
    "    text = text.decode(\"utf-8\")\n",
    "    truncated = text[:cut_length] if cut_length is not None else text\n",
    "    max_x = np.ceil(np.sqrt(len(truncated))).astype(\"int\")\n",
    "    character_size = np.floor(size / max_x).astype(\"int\")\n",
    "    padding = np.floor((size - (max_x * character_size)) / 2).astype(\"int\")\n",
    "    # Do we need pt to px conversion ? Seems like not\n",
    "    # font_size =  int(np.floor(character_size*0.75))\n",
    "    font_size = character_size\n",
    "\n",
    "    fnt = ImageFont.truetype(out_font.as_posix(), font_size)\n",
    "\n",
    "    # 1 (1-bit pixels, black and white, stored with one pixel per byte)\n",
    "    # L (8-bit pixels, black and white)\n",
    "    # RGB (3x8-bit pixels, true color)\n",
    "    # https://pillow.readthedocs.io/en/stable/handbook/concepts.html#concept-modes\n",
    "    mode = \"L\" if one_channel else \"RGB\"\n",
    "    WHITE = 1 if one_channel else (255, 255, 255)\n",
    "    BLACK = 0 if one_channel else (0, 0, 0)\n",
    "\n",
    "    image = Image.new(mode, (size, size), BLACK)\n",
    "    # Obtention du contexte graphique\n",
    "    draw = ImageDraw.Draw(image)\n",
    "    x = 0\n",
    "    y = 0\n",
    "    for letter in truncated:\n",
    "        draw.text(\n",
    "            (padding + x * character_size, padding + y * character_size),\n",
    "            letter,\n",
    "            font=fnt,\n",
    "            fill=WHITE,\n",
    "        )\n",
    "        if x + 1 < max_x:\n",
    "            x += 1\n",
    "        else:\n",
    "            y += 1\n",
    "            x = 0\n",
    "    return np.array(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.511814Z",
     "start_time": "2020-04-29T16:51:44.490168Z"
    }
   },
   "outputs": [],
   "source": [
    "def features_to_square_image(\n",
    "    features, image_size=224, cut_length=None, one_channel=False\n",
    "):\n",
    "    nb_channel = 1 if one_channel else 3\n",
    "    square_nb = np.ceil(np.sqrt(len(features))).astype(\"int\")\n",
    "    word_size = np.floor(image_size / square_nb).astype(\"int\")\n",
    "    max_features = len(features)\n",
    "    padding = np.floor((image_size - square_nb * word_size) / 2).astype(\"int\")\n",
    "    if one_channel:\n",
    "        result_image = np.zeros((image_size, image_size), dtype=\"uint8\")\n",
    "    else:\n",
    "        result_image = np.zeros((image_size, image_size, nb_channel), dtype=\"uint8\")\n",
    "    results = []\n",
    "    i_feature = 0\n",
    "    features_str = features  # .astype(\"str\")\n",
    "    for x in range(0, square_nb):\n",
    "        if i_feature is None:\n",
    "            break\n",
    "        for y in range(0, square_nb):\n",
    "            i_feature = x * (square_nb) + y\n",
    "            if i_feature >= max_features:\n",
    "                i_feature = None\n",
    "                break\n",
    "            x_pos = x * word_size + padding\n",
    "            y_pos = y * word_size + padding\n",
    "            result_image[\n",
    "                x_pos : x_pos + word_size, y_pos : y_pos + word_size\n",
    "            ] = word_to_square_image(\n",
    "                features_str[i_feature],\n",
    "                size=word_size,\n",
    "                cut_length=cut_length,\n",
    "                one_channel=one_channel,\n",
    "            )\n",
    "    return result_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.518044Z",
     "start_time": "2020-04-29T16:51:44.512730Z"
    }
   },
   "outputs": [],
   "source": [
    "def features_to_square_image_params(X, Y):\n",
    "    return (\n",
    "        features_to_square_image(\n",
    "            X, image_size=IMAGE_SIZE, cut_length=CUT_LENGTH, one_channel=ONE_CHANNEL,\n",
    "        ),\n",
    "        CLASSNAMES == Y,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.529129Z",
     "start_time": "2020-04-29T16:51:44.518800Z"
    }
   },
   "outputs": [],
   "source": [
    "@tf.function(\n",
    "    input_signature=[tf.TensorSpec(None, tf.string), tf.TensorSpec(None, tf.string)]\n",
    ")\n",
    "def tf_features_to_square_image_params(X, Y):\n",
    "    img, label = tf.numpy_function(\n",
    "        features_to_square_image_params, [X, Y], (tf.uint8, tf.bool),\n",
    "    )\n",
    "    return (\n",
    "        tf.reshape(img, shape=(IMAGE_SIZE, IMAGE_SIZE, NB_CHANNEL)),\n",
    "        tf.reshape(label, shape=(len(CLASSNAMES),)),\n",
    "    )"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T15:15:48.637798Z",
     "start_time": "2020-04-29T15:15:48.615364Z"
    }
   },
   "source": [
    "def my_numpy_func(x): \n",
    "  # x will be a numpy array with the contents of the input to the \n",
    "  # tf.function \n",
    "  return np.sinh(x)\n",
    "\n",
    "@tf.function(input_signature=[tf.TensorSpec(None, tf.float32)]) \n",
    "def tf_function(input): \n",
    "  y = tf.numpy_function(my_numpy_func, [input], tf.float32) \n",
    "  return y * y \n",
    "\n",
    "    train_img = tf.reshape(train_img, shape=(input_width, input_height, input_channel))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:44.540431Z",
     "start_time": "2020-04-29T16:51:44.529960Z"
    }
   },
   "outputs": [],
   "source": [
    "AUTOTUNE = tf.data.experimental.AUTOTUNE\n",
    "\n",
    "\n",
    "def build_dataset(X, Y, *, repeat, batch_size, prefetch):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((X, Y))\n",
    "    if repeat:\n",
    "        dataset = dataset.repeat()\n",
    "\n",
    "    dataset = dataset.map(\n",
    "        tf_features_to_square_image_params, num_parallel_calls=AUTOTUNE\n",
    "    )\n",
    "\n",
    "    if batch_size is not None:\n",
    "        dataset = dataset.batch(batch_size)\n",
    "\n",
    "    if prefetch is not None:\n",
    "        dataset = dataset.prefetch(prefetch)\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:45.107802Z",
     "start_time": "2020-04-29T16:51:44.541230Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_train = build_dataset(\n",
    "    X_train, Y_train, repeat=True, batch_size=BATCH_SIZE, prefetch=PREFETCH\n",
    ")\n",
    "dataset_valid = build_dataset(\n",
    "    X_valid, Y_valid, repeat=True, batch_size=BATCH_SIZE, prefetch=PREFETCH\n",
    ")\n",
    "dataset_test = build_dataset(\n",
    "    X_test, Y_test, repeat=False, batch_size=BATCH_SIZE, prefetch=PREFETCH\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:45.114775Z",
     "start_time": "2020-04-29T16:51:45.109868Z"
    }
   },
   "outputs": [],
   "source": [
    "def show_image(image, image_size, one_channel=False):\n",
    "    if one_channel:\n",
    "        imshow(image.reshape(IMAGE_SIZE, IMAGE_SIZE))\n",
    "    else:\n",
    "        imshow(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:46.723911Z",
     "start_time": "2020-04-29T16:51:45.115687Z"
    }
   },
   "outputs": [],
   "source": [
    "for image, label in dataset_train.take(1):\n",
    "    print(\"Label: \", label[0].numpy())\n",
    "    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:47.530692Z",
     "start_time": "2020-04-29T16:51:46.724717Z"
    }
   },
   "outputs": [],
   "source": [
    "for image, label in dataset_valid.take(1):\n",
    "    print(\"Label: \", label[0].numpy())\n",
    "    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:48.534429Z",
     "start_time": "2020-04-29T16:51:47.531621Z"
    }
   },
   "outputs": [],
   "source": [
    "for image, label in dataset_test.take(1):\n",
    "    print(\"Label: \", label[0].numpy())\n",
    "    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:48.557421Z",
     "start_time": "2020-04-29T16:51:48.535312Z"
    }
   },
   "outputs": [],
   "source": [
    "from efficientnet.tfkeras import (\n",
    "    EfficientNetB0,\n",
    "    EfficientNetB4,\n",
    "    EfficientNetB2,\n",
    "    EfficientNetB3,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:48.562599Z",
     "start_time": "2020-04-29T16:51:48.558331Z"
    }
   },
   "outputs": [],
   "source": [
    "activation = mish\n",
    "optimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:49.994269Z",
     "start_time": "2020-04-29T16:51:48.563350Z"
    }
   },
   "outputs": [],
   "source": [
    "# Now, we can use PRE TRAINED model\n",
    "base_model = EfficientNetB0(\n",
    "    input_shape=(IMAGE_SIZE, IMAGE_SIZE, NB_CHANNEL),\n",
    "    weights=None,  # \"imagenet\",\n",
    "    # weights=\"noisy-student\",\n",
    "    # weights=None,\n",
    "    include_top=False,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:50.086801Z",
     "start_time": "2020-04-29T16:51:49.995227Z"
    }
   },
   "outputs": [],
   "source": [
    "x = base_model.layers[-4].output\n",
    "# add a global spatial average pooling layer\n",
    "x = GlobalAveragePooling2D()(x)\n",
    "# let's add a fully-connected layer\n",
    "# x = Dense(1024, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "# x = Dense(512, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "x = Dense(256, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "x = Dropout(0.2)(x)\n",
    "x = Dense(128, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "x = Dropout(0.2)(x)\n",
    "# and a logistic layer -- let's say we have 200 classes\n",
    "predictions = Dense(OUTPUT_DIM, activation=\"softmax\")(x)\n",
    "\n",
    "# this is the model we will train\n",
    "model = Model(inputs=base_model.input, outputs=predictions)\n",
    "\n",
    "# first: train only the top layers (which were randomly initialized)\n",
    "# i.e. freeze all base_model layers\n",
    "# for layer in base_model.layers:\n",
    "#    layer.trainable = False\n",
    "\n",
    "# compile the model (should be done *after* setting layers to non-trainable)\n",
    "es = EarlyStopping(\n",
    "    monitor=\"val_loss\",\n",
    "    verbose=1,\n",
    "    mode=\"min\",\n",
    "    patience=patience,\n",
    "    restore_best_weights=True,\n",
    ")\n",
    "# We need to recompile the model for these modifications to take effect\n",
    "es.set_model(model)\n",
    "model.compile(optimizer=optimizer, loss=LOSS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:50.135283Z",
     "start_time": "2020-04-29T16:51:50.087771Z"
    }
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T16:51:50.139773Z",
     "start_time": "2020-04-29T16:51:50.136115Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T17:17:04.768403Z",
     "start_time": "2020-04-29T16:51:50.140554Z"
    }
   },
   "outputs": [],
   "source": [
    "# we train our model again (this time fine-tuning the top 2 inception blocks\n",
    "# alongside the top Dense layers\n",
    "history = model.fit(\n",
    "    dataset_train,\n",
    "    callbacks=[es],\n",
    "    epochs=epochs,\n",
    "    steps_per_epoch=steps_per_epoch,\n",
    "    validation_data=dataset_valid,\n",
    "    validation_steps=steps_per_epoch_val,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T17:17:05.075866Z",
     "start_time": "2020-04-29T16:51:42.432Z"
    }
   },
   "outputs": [],
   "source": [
    "truth_test = []\n",
    "for _, labels in dataset_test:\n",
    "    truth_test.append(np.argmax(labels, axis=1))\n",
    "truth_test = np.hstack(truth_test)\n",
    "truth_test\n",
    "\n",
    "truth_valid = []\n",
    "for i, (_, labels) in enumerate(dataset_valid):\n",
    "    truth_valid.append(np.argmax(labels, axis=1))\n",
    "    if i >= steps_per_epoch_val - 1:\n",
    "        break\n",
    "truth_valid = np.hstack(truth_valid)\n",
    "truth_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-29T17:17:05.153542Z",
     "start_time": "2020-04-29T16:51:42.434Z"
    }
   },
   "outputs": [],
   "source": [
    "preds_valid = model.predict(dataset_valid, steps=steps_per_epoch_val)\n",
    "preds_valid.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(\n",
    "        f\"Accuracy valid: {accuracy_score(truth_valid, np.argmax(preds_valid, axis=1))}\"\n",
    "    )\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC valid: {roc_auc_score(truth_valid, preds_valid[:, 1])}\")\n",
    "preds_test = model.predict(dataset_test)\n",
    "preds_test.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(f\"Accuracy test: {accuracy_score(truth_test, np.argmax(preds_test, axis=1))}\")\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC test: {roc_auc_score(truth_test, preds_test[:, 1])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "notify_time": "5"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}