Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.preprocessing.image import load_img,img_to_array\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from keras.initializers import RandomNormal\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.optimizers import SGD\n",
    "from keras.models import Model,Sequential\n",
    "from keras.layers import *\n",
    "from keras import backend as K\n",
    "from keras.models import model_from_json\n",
    "from matplotlib import cm as CM\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "import scipy.io as io\n",
    "from PIL import Image\n",
    "import PIL\n",
    "import h5py\n",
    "import os\n",
    "import glob\n",
    "import cv2\n",
    "import random\n",
    "import math\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/rana/mypython/crowd_counting_analysis/keras_solution/CSRNet_keras_master/CSRNet_keras_master/ShanghaiTech\n"
     ]
    }
   ],
   "source": [
    "K.clear_session()\n",
    "root = os.path.join(os.getcwd(),'ShanghaiTech')\n",
    "print(root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "part_A_train = os.path.join(root,'part_A/train_data','images')\n",
    "part_A_test = os.path.join(root,'part_A/test_data','images')\n",
    "part_B_train = os.path.join(root,'part_B/train_data','images')\n",
    "part_B_test = os.path.join(root,'part_B/test_data','images')\n",
    "temp = 'test_images'\n",
    "path_sets = [part_A_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total images :  300\n"
     ]
    }
   ],
   "source": [
    "img_paths = []\n",
    "\n",
    "for path in path_sets:\n",
    "    \n",
    "    for img_path in glob.glob(os.path.join(path, '*.jpg')):\n",
    "        \n",
    "        img_paths.append(str(img_path))\n",
    "        \n",
    "print(\"Total images : \",len(img_paths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_img(path):\n",
    "    #Function to load,normalize and return image \n",
    "    im = Image.open(path).convert('RGB')\n",
    "    \n",
    "    im = np.array(im)\n",
    "    \n",
    "    im = im/255.0\n",
    "    \n",
    "    im[:,:,0]=(im[:,:,0]-0.485)/0.229\n",
    "    im[:,:,1]=(im[:,:,1]-0.456)/0.224\n",
    "    im[:,:,2]=(im[:,:,2]-0.406)/0.225\n",
    "\n",
    "    #print(im.shape)\n",
    "    #im = np.expand_dims(im,axis  = 0)\n",
    "    return im\n",
    "\n",
    "def get_input(path):\n",
    "    path = path[0] \n",
    "    img = create_img(path)\n",
    "    return(img)\n",
    "    \n",
    "    \n",
    "    \n",
    "def get_output(path):\n",
    "    #import target\n",
    "    #resize target\n",
    "    \n",
    "    gt_file = h5py.File(path,'r')\n",
    "    \n",
    "    target = np.asarray(gt_file['density'])\n",
    "    \n",
    "    img = cv2.resize(target,(int(target.shape[1]/8),int(target.shape[0]/8)),interpolation = cv2.INTER_CUBIC)*64\n",
    "    \n",
    "    img = np.expand_dims(img,axis  = 3)\n",
    "    \n",
    "    #print(img.shape)\n",
    "    \n",
    "    return img\n",
    "    \n",
    "    \n",
    "    \n",
    "def preprocess_input(image,target):\n",
    "    #crop image\n",
    "    #crop target\n",
    "    #resize target\n",
    "    crop_size = (int(image.shape[0]/2),int(image.shape[1]/2))\n",
    "    \n",
    "    \n",
    "    if random.randint(0,9)<= -1:            \n",
    "            dx = int(random.randint(0,1)*image.shape[0]*1./2)\n",
    "            dy = int(random.randint(0,1)*image.shape[1]*1./2)\n",
    "    else:\n",
    "            dx = int(random.random()*image.shape[0]*1./2)\n",
    "            dy = int(random.random()*image.shape[1]*1./2)\n",
    "\n",
    "    #print(crop_size , dx , dy)\n",
    "    img = image[dx : crop_size[0]+dx , dy:crop_size[1]+dy]\n",
    "    \n",
    "    target_aug = target[dx:crop_size[0]+dx,dy:crop_size[1]+dy]\n",
    "    #print(img.shape)\n",
    "\n",
    "    return(img,target_aug)\n",
    "    \n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Image data generator \n",
    "def image_generator(files, batch_size = 64):\n",
    "    \n",
    "    while True:\n",
    "        \n",
    "        input_path = np.random.choice(a = files, size = batch_size)\n",
    "        \n",
    "        batch_input = []\n",
    "        batch_output = [] \n",
    "          \n",
    "        #for input_path in batch_paths:\n",
    "        \n",
    "        inputt = get_input(input_path )\n",
    "        output = get_output(input_path[0].replace('.jpg','.h5').replace('images','ground_truth') )\n",
    "            \n",
    "       \n",
    "        batch_input += [inputt]\n",
    "        batch_output += [output]\n",
    "    \n",
    "\n",
    "        batch_x = np.array( batch_input )\n",
    "        batch_y = np.array( batch_output )\n",
    "        \n",
    "        yield( batch_x, batch_y )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_mod(model , str1 , str2):\n",
    "    model.save_weights(str1)\n",
    "    \n",
    "    model_json = model.to_json()\n",
    "    \n",
    "    with open(str2, \"w\") as json_file:\n",
    "        json_file.write(model_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights_vgg(model):\n",
    "    #vgg =  VGG16(weights='imagenet', include_top=False)\n",
    "    \n",
    "    json_file = open('models/VGG_16.json', 'r')\n",
    "    loaded_model_json = json_file.read()\n",
    "    json_file.close()\n",
    "    loaded_model = model_from_json(loaded_model_json)\n",
    "    loaded_model.load_weights(\"weights/VGG_16.h5\")\n",
    "    \n",
    "    vgg = loaded_model\n",
    "    \n",
    "    vgg_weights=[]                         \n",
    "    for layer in vgg.layers:\n",
    "        if('conv' in layer.name):\n",
    "            vgg_weights.append(layer.get_weights())\n",
    "    \n",
    "    \n",
    "    offset=0\n",
    "    i=0\n",
    "    while(i<10):\n",
    "        if('conv' in model.layers[i+offset].name):\n",
    "            model.layers[i+offset].set_weights(vgg_weights[i])\n",
    "            i=i+1\n",
    "            #print('h')\n",
    "            \n",
    "        else:\n",
    "            offset=offset+1\n",
    "\n",
    "    return (model)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def euclidean_distance_loss(y_true, y_pred):\n",
    "    # Euclidean distance as a measure of loss (Loss function) \n",
    "    return K.sqrt(K.sum(K.square(y_pred - y_true), axis=-1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Neural network model : VGG + Conv\n",
    "def CrowdNet():  \n",
    "            #Variable Input Size\n",
    "            rows = None\n",
    "            cols = None\n",
    "            \n",
    "            #Batch Normalisation option\n",
    "            \n",
    "            batch_norm = 0\n",
    "            kernel = (3, 3)\n",
    "            init = RandomNormal(stddev=0.01)\n",
    "            model = Sequential() \n",
    "            \n",
    "            #custom VGG:\n",
    "            \n",
    "            if(batch_norm):\n",
    "                model.add(Conv2D(64, kernel_size = kernel, input_shape = (rows,cols,3),activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                model.add(Conv2D(64, kernel_size = kernel,activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                model.add(MaxPooling2D(strides=2))\n",
    "                model.add(Conv2D(128,kernel_size = kernel, activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                model.add(Conv2D(128,kernel_size = kernel, activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                model.add(MaxPooling2D(strides=2))\n",
    "                model.add(Conv2D(256,kernel_size = kernel, activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                model.add(Conv2D(256,kernel_size = kernel, activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                model.add(Conv2D(256,kernel_size = kernel, activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                model.add(MaxPooling2D(strides=2))            \n",
    "                model.add(Conv2D(512, kernel_size = kernel,activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                model.add(Conv2D(512, kernel_size = kernel,activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                model.add(Conv2D(512, kernel_size = kernel,activation = 'relu', padding='same'))\n",
    "                model.add(BatchNormalization())\n",
    "                \n",
    "            else:\n",
    "                model.add(Conv2D(64, kernel_size = kernel,activation = 'relu', padding='same',input_shape = (rows, cols, 3), kernel_initializer = init))\n",
    "                model.add(Conv2D(64, kernel_size = kernel,activation = 'relu', padding='same', kernel_initializer = init))\n",
    "                model.add(MaxPooling2D(strides=2))\n",
    "                model.add(Conv2D(128,kernel_size = kernel, activation = 'relu', padding='same', kernel_initializer = init))\n",
    "                model.add(Conv2D(128,kernel_size = kernel, activation = 'relu', padding='same', kernel_initializer = init))\n",
    "                model.add(MaxPooling2D(strides=2))\n",
    "                model.add(Conv2D(256,kernel_size = kernel, activation = 'relu', padding='same', kernel_initializer = init))\n",
    "                model.add(Conv2D(256,kernel_size = kernel, activation = 'relu', padding='same', kernel_initializer = init))\n",
    "                model.add(Conv2D(256,kernel_size = kernel, activation = 'relu', padding='same', kernel_initializer = init))\n",
    "                model.add(MaxPooling2D(strides=2))            \n",
    "                model.add(Conv2D(512, kernel_size = kernel,activation = 'relu', padding='same', kernel_initializer = init))\n",
    "                model.add(Conv2D(512, kernel_size = kernel,activation = 'relu', padding='same', kernel_initializer = init))\n",
    "                model.add(Conv2D(512, kernel_size = kernel,activation = 'relu', padding='same', kernel_initializer = init))\n",
    "                \n",
    "                \n",
    "\n",
    "                \n",
    "            #Conv2D\n",
    "            model.add(Conv2D(512, (3, 3), activation='relu', dilation_rate = 2, kernel_initializer = init, padding = 'same'))\n",
    "            model.add(Conv2D(512, (3, 3), activation='relu', dilation_rate = 2, kernel_initializer = init, padding = 'same'))\n",
    "            model.add(Conv2D(512, (3, 3), activation='relu', dilation_rate = 2, kernel_initializer = init, padding = 'same'))\n",
    "            model.add(Conv2D(256, (3, 3), activation='relu', dilation_rate = 2, kernel_initializer = init, padding = 'same'))\n",
    "            model.add(Conv2D(128, (3, 3), activation='relu', dilation_rate = 2, kernel_initializer = init, padding = 'same'))\n",
    "            model.add(Conv2D(64, (3, 3), activation='relu', dilation_rate = 2, kernel_initializer = init, padding = 'same'))\n",
    "            model.add(Conv2D(1, (1, 1), activation='relu', dilation_rate = 1, kernel_initializer = init, padding = 'same'))\n",
    "        \n",
    "            sgd = SGD(lr = 1e-7, decay = (5*1e-4), momentum = 0.95)\n",
    "            model.compile(optimizer=sgd, loss=euclidean_distance_loss, metrics=['mse'])\n",
    "            \n",
    "            model = init_weights_vgg(model)\n",
    "            \n",
    "            return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CrowdNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, None, None, 64)    1792      \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, None, None, 64)    36928     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, None, None, 64)    0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, None, None, 128)   73856     \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, None, None, 128)   147584    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, None, None, 128)   0         \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, None, None, 256)   295168    \n",
      "_________________________________________________________________\n",
      "conv2d_6 (Conv2D)            (None, None, None, 256)   590080    \n",
      "_________________________________________________________________\n",
      "conv2d_7 (Conv2D)            (None, None, None, 256)   590080    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, None, None, 256)   0         \n",
      "_________________________________________________________________\n",
      "conv2d_8 (Conv2D)            (None, None, None, 512)   1180160   \n",
      "_________________________________________________________________\n",
      "conv2d_9 (Conv2D)            (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "conv2d_10 (Conv2D)           (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "conv2d_11 (Conv2D)           (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "conv2d_12 (Conv2D)           (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "conv2d_13 (Conv2D)           (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, None, None, 256)   1179904   \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, None, None, 128)   295040    \n",
      "_________________________________________________________________\n",
      "conv2d_16 (Conv2D)           (None, None, None, 64)    73792     \n",
      "_________________________________________________________________\n",
      "conv2d_17 (Conv2D)           (None, None, None, 1)     65        \n",
      "=================================================================\n",
      "Total params: 16,263,489\n",
      "Trainable params: 16,263,489\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_gen = image_generator(img_paths,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd = SGD(lr = 1e-7, decay = (5*1e-4), momentum = 0.95)\n",
    "model.compile(optimizer=sgd, loss=euclidean_distance_loss, metrics=['mse'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rana/anaconda3/envs/conaenv/lib/python3.5/site-packages/ipykernel_launcher.py:34: DeprecationWarning: Both axis > a.ndim and axis < -a.ndim - 1 are deprecated and will raise an AxisError in the future.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "159/700 [=====>........................] - ETA: 3:34:51 - loss: nan - mean_squared_error: nan"
     ]
    }
   ],
   "source": [
    "model.fit_generator(train_gen,epochs=15,steps_per_epoch= 700 , verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_mod(model,\"weights/model_A_weights.h5\",\"models/Model.json\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}