Mask_R_CNN_Dataset_Generator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JvlLvOyBJTrI"
   },
   "source": [
    "# Dataset Generator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OBEUiMgxJQ8-"
   },
   "source": [
    "## Configuration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ks0oMHRGJtdi"
   },
   "source": [
    "### Customizing names of folders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5JnvH66cKVcd"
   },
   "source": [
    "If you want to change names of the directories that will be created."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PvHQfzDlG4L1"
   },
   "outputs": [],
   "source": [
    "datasetToGenerate = \"regenerate\" #@param [\"cortex\", \"main\", \"mest_main\", \"mest_glom\", \"inflammation\", \"regenerate\"]\n",
    "\n",
    "# Path to the file with info to recreate the same dataset\n",
    "recreateValDatasetFilePath = \"dataset.json\" #@param {type:\"string\"}\n",
    "\n",
    "# Name of the folder where to found base images and annotations\n",
    "rawDataset = \"raw_dataset\" #@param {type:\"string\"}\n",
    "# Name of the folder that will be used to store temporary needed files\n",
    "tempDataset = \"temp_dataset\" #@param {type:\"string\"}\n",
    "# Name that will be used to create folders for training and validation of the \n",
    "# cortex training\n",
    "cortexDatasetPath = \"nephrology_cortex_dataset\" #@param {type:\"string\"}\n",
    "mestDatasetPath = f\"nephrology_{datasetToGenerate}_dataset\" #@param {type:\"string\"}\n",
    "# Name of the folder that will contain every images and division that cannot be used\n",
    "unusedDirPath = \"nephrology_dataset_unused\" #@param {type:\"string\"}\n",
    "\n",
    "# Name that will be used to create folders for training and validation of the \n",
    "# main training\n",
    "mainDataset = \"nephrology_dataset\" #@param {type:\"string\"}\n",
    "# Name of the folder for files that cannot be used for the main training\n",
    "mainDatasetUnusedDirPath = \"nephrology_dataset_unused\" #@param {type:\"string\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ImKpEscJJRVe"
   },
   "source": [
    "### Customizing reading and dividing of images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "j967V6tKS6Yb"
   },
   "source": [
    "Here, you can change the division size and minimum overlap. You can also force the generator to use a specific annotation format if there are different ones in the raw directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aJIPwDR5ItnM"
   },
   "outputs": [],
   "source": [
    "# If you want to force one annotation format to be read :\n",
    "# 1) Import the AnnotationAdapter you want\n",
    "# 2) Set the `adapter` variable to a reference of the imported adapter\n",
    "from datasetTools.AnnotationAdapter import *\n",
    "from datasetTools.ASAPAdapter import ASAPAdapter\n",
    "from datasetTools.LabelMeAdapter import LabelMeAdapter\n",
    "adapter = None #@param [\"ASAPAdapter\", \"LabelMeAdapter\", \"None\"] {type:\"raw\", allow-input: true}\n",
    "\n",
    "# The image format to use for the datasets\n",
    "imageFormat = \"jpg\" #@param [\"jpg\", \"png\"]\n",
    "\n",
    "# Side size for the divisions that will be created\n",
    "divisionSize = 1024 #@param {type:\"slider\", min:896, max:1024, step:1}\n",
    "\n",
    "# Minimum overlap between divisions between 0.0 and 1.0\n",
    "minDivisionOverlap = 0.33 #@param {type:\"slider\", min:0, max:1, step:0.01}\n",
    "\n",
    "cortexSize = 2048 #@param {type:\"slider\", min:1024, max:4096, step:1}\n",
    "cortexResize = (cortexSize, cortexSize)\n",
    "\n",
    "# Minimum overlap between cortex divisions between 0.0 and 1.0\n",
    "minCortexDivisionOverlap = 0.00 #@param {type:\"slider\", min:0, max:1, step:0.01}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AuuYuTYrJR9x"
   },
   "source": [
    "### Customizing generator's behaviour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PyrLE92sIiDn"
   },
   "outputs": [],
   "source": [
    "# Set to True if you want the previous directories to be deleted, \n",
    "# keeping only new ones\n",
    "cleanBeforeStart = False #@param {type:\"boolean\"}\n",
    "\n",
    "# When creating masks, if an image has more than one cortex mask, they will be \n",
    "# fused. Setting this to True will delete the original masks and keep only the \n",
    "# fused one.\n",
    "deleteBaseCortexMasks = True #@param {type:\"boolean\"}\n",
    "\n",
    "# If False, some images will be separated from the dataset before being divided\n",
    "# to create the validation dataset. This avoid divisions from the same image\n",
    "# being in training and validation dataset.\n",
    "# If True, images will be divided, then some of the divisions will be separated \n",
    "# to create the validation dataset. \n",
    "separateDivInsteadOfImage = False #@param {type:\"boolean\"}\n",
    "\n",
    "# If True and separateDivInsteadOfImage is False, will select some patients from\n",
    "# the dataset and create the val dataset with all the images of these patients\n",
    "separateByPatient = True #@param {type:\"boolean\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rJheyCReJSul"
   },
   "source": [
    "## Generation of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FwocTeMsGZCr"
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    from datasetTools import datasetFormator as df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FwocTeMsGZCr",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if datasetToGenerate == \"main\":\n",
    "    df.generateDataset(\n",
    "        rawDataset=rawDataset, tempDataset=tempDataset, unusedDirPath=unusedDirPath,\n",
    "        mainDataset=mainDataset, mainDatasetUnusedDirPath=mainDatasetUnusedDirPath,\n",
    "        deleteBaseCortexMasks=deleteBaseCortexMasks, cleanBeforeStart=cleanBeforeStart,\n",
    "        recreateValList=None, divisionSize=divisionSize, imageFormat=imageFormat,\n",
    "        separateDivInsteadOfImage=separateDivInsteadOfImage, adapter=adapter,\n",
    "        separateByPatient=separateByPatient, minDivisionOverlapping=minDivisionOverlap\n",
    "    )\n",
    "elif datasetToGenerate == \"cortex\":\n",
    "    df.generateCortexDataset(\n",
    "        rawDataset=rawDataset, outputDataset=cortexDatasetPath,\n",
    "        cleanBeforeStart=cleanBeforeStart, resize=cortexResize,\n",
    "        overlap=minCortexDivisionOverlap, recreateValList=None,\n",
    "        separateDivInsteadOfImage=separateDivInsteadOfImage\n",
    "    )\n",
    "elif \"mest_\" in datasetToGenerate:\n",
    "    submode = datasetToGenerate.split('_')[-1]\n",
    "    separateMode = \"div\" if separateDivInsteadOfImage else (\"patient\" if separateByPatient else \"images\")\n",
    "    df.generateMESTCDataset(\n",
    "        rawDataset=rawDataset, outputDataset=mestDatasetPath, mode=submode,\n",
    "        cleanBeforeStart=cleanBeforeStart, imageFormat=imageFormat,\n",
    "        divisionSize=divisionSize, overlap=minDivisionOverlap,\n",
    "        separate=separateMode, adapter=adapter\n",
    "    )\n",
    "elif datasetToGenerate == \"inflammation\":\n",
    "    separateMode = \"div\" if separateDivInsteadOfImage else (\"patient\" if separateByPatient else \"images\")\n",
    "    df.generateInflammationDataset(\n",
    "        rawDataset=rawDataset, outputDataset=mestDatasetPath,\n",
    "        cleanBeforeStart=cleanBeforeStart, imageFormat=imageFormat,\n",
    "        divisionSize=divisionSize, overlap=minDivisionOverlap,\n",
    "        separate=separateMode, adapter=adapter\n",
    "    )\n",
    "elif datasetToGenerate == \"regenerate\":\n",
    "    df.regenerateDataset(\n",
    "        rawDataset=rawDataset, adapter=adapter,\n",
    "        recreateFilePath=recreateValDatasetFilePath\n",
    "    )\n",
    "print(\"##### END #####\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyP/DR7bqOAdJkRFJPyy8SHN",
   "collapsed_sections": [],
   "mount_file_id": "1wlL93qsDBT76FfyYEOUcHeA4A763tMJ4",
   "name": "Mask R-CNN - Dataset Generator",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}