brainiac/data_preprocessing/data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pydicom import dcmread\n",
    "from os import listdir\n",
    "from os.path import isfile, join, isdir\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change file path for appropriate one\n",
    "base_path = '/home/ADNI/'\n",
    "csv_path = '/home/ADNI/3T_3D_1mm_4_05_2020.csv'\n",
    "\n",
    "save_path = '/home/basimova_nf/ADNI-processed/images/'\n",
    "csv_save_path = '/home/basimova_nf/ADNI-processed/data.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_file_names = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# based on data structure\n",
    "for folder1 in listdir(base_path):\n",
    "    if isdir(f'{base_path}{folder1}'):\n",
    "        for folder2 in listdir(f'{base_path}{folder1}'):\n",
    "            for folder3 in listdir(f'{base_path}{folder1}/{folder2}'):\n",
    "                for folder4 in listdir(f'{base_path}{folder1}/{folder2}/{folder3}'):\n",
    "                    one_picture_files = []\n",
    "                    for file_name in listdir(f'{base_path}{folder1}/{folder2}/{folder3}/{folder4}'):\n",
    "                        one_picture_files.append(f'{base_path}{folder1}/{folder2}/{folder3}/{folder4}/{file_name}')\n",
    "                    all_file_names.append(one_picture_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_files = []\n",
    "for i, one_picture_files in enumerate(all_file_names):\n",
    "    sorted_files.append(sorted(one_picture_files, key=lambda string: int(string.split('_')[-3])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_csv = pd.read_csv(csv_path)\n",
    "# data_csv.drop(columns=['Unnamed: 0'], inplace=True)\n",
    "data_csv['File Names'] = data_csv['Group']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_file_id(file_name):\n",
    "    ID = file_name.split('_')[-1][1:-4]\n",
    "    return int(ID)\n",
    "def make_good_file_name(full_file_name):\n",
    "    return '-'.join(full_file_name.split('/')[4:8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_files_id = set()\n",
    "bad_folders = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# will not use bad small pictures\n",
    "min_images_number = 174"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for one_folder in sorted_files:\n",
    "    print('____')\n",
    "    try:\n",
    "        if len(one_folder) >= min_images_number:\n",
    "            ID = get_file_id(one_folder[0])\n",
    "            base_shape = dcmread(one_folder[0]).pixel_array.shape\n",
    "            img = np.zeros((len(one_folder), *base_shape))\n",
    "            for i, file in enumerate(one_folder):\n",
    "                print(i/len(one_folder))\n",
    "                dcm_file = dcmread(file)\n",
    "                img[i] = dcm_file.pixel_array\n",
    "            np.save(save_path+make_good_file_name(file), img)\n",
    "            data_csv.loc[data_csv['Image Data ID']==ID, 'File Names'] = make_good_file_name(file)\n",
    "        else:\n",
    "            bad_folders.append(one_folder)\n",
    "            if len(one_folder) > 0: bad_files_id.add(get_file_id(one_folder[0]))\n",
    "    except:\n",
    "        bad_folders.append(one_folder)\n",
    "        bad_files_id.add(get_file_id(file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_csv = data_csv[~data_csv['Image Data ID'].isin(bad_files_id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_csv.to_csv(csv_save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(img[img.shape[0]//2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}