00_ladder-network_lending.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-30T09:56:19.102265Z",
     "start_time": "2018-01-30T09:56:17.730310Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import math\n",
    "import os\n",
    "import csv\n",
    "from tqdm import tqdm\n",
    "import numpy\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   \n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "CHECK_POINT_PATH = \"checkpoints_lending\"\n",
    "LOG_FILE_PATH = \"train_log_lending\"\n",
    "CONTINUE = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-30T09:56:19.841185Z",
     "start_time": "2018-01-30T09:56:19.119031Z"
    }
   },
   "outputs": [],
   "source": [
    "def bi(inits, size, name):\n",
    "    return tf.Variable(inits * tf.ones([size]), name=name)\n",
    "\n",
    "\n",
    "def wi(shape, name):\n",
    "    return tf.Variable(tf.random_normal(shape, name=name)) / math.sqrt(shape[0])\n",
    "\n",
    "def batch_normalization(batch, mean=None, var=None):\n",
    "    if mean is None or var is None:\n",
    "        mean, var = tf.nn.moments(batch, axes=[0])\n",
    "    return (batch - mean) / tf.sqrt(var + tf.constant(1e-10))\n",
    "\n",
    "def update_batch_normalization(batch, l):\n",
    "    \"batch normalize + update average mean and variance of layer l\"\n",
    "    mean, var = tf.nn.moments(batch, axes=[0])\n",
    "    assign_mean = running_mean[l-1].assign(mean)\n",
    "    assign_var = running_var[l-1].assign(var)\n",
    "    bn_assigns.append(ewma.apply([running_mean[l-1], running_var[l-1]]))\n",
    "    with tf.control_dependencies([assign_mean, assign_var]):\n",
    "        return (batch - mean) / tf.sqrt(var + 1e-10)\n",
    "\n",
    "\n",
    "def encoder(inputs, noise_std):\n",
    "    h = inputs + tf.random_normal(tf.shape(inputs)) * noise_std  # add noise to input\n",
    "    d = {}  # to store the pre-activation, activation, mean and variance for each layer\n",
    "    # The data for labeled and unlabeled examples are stored separately\n",
    "    d['labeled'] = {'z': {}, 'm': {}, 'v': {}, 'h': {}}\n",
    "    d['unlabeled'] = {'z': {}, 'm': {}, 'v': {}, 'h': {}}\n",
    "    d['labeled']['z'][0], d['unlabeled']['z'][0] = split_lu(h)\n",
    "    for l in range(1, L+1):\n",
    "        print(\"Layer \", l, \": \", layer_sizes[l-1], \" -> \", layer_sizes[l])\n",
    "        d['labeled']['h'][l-1], d['unlabeled']['h'][l-1] = split_lu(h)\n",
    "        z_pre = tf.matmul(h, weights['W'][l-1])  # pre-activation\n",
    "        z_pre_l, z_pre_u = split_lu(z_pre)  # split labeled and unlabeled examples\n",
    "\n",
    "        m, v = tf.nn.moments(z_pre_u, axes=[0])\n",
    "\n",
    "        # if training:\n",
    "        def training_batch_norm():\n",
    "            # Training batch normalization\n",
    "            # batch normalization for labeled and unlabeled examples is performed separately\n",
    "            if noise_std > 0:\n",
    "                # Corrupted encoder\n",
    "                # batch normalization + noise\n",
    "                z = join(batch_normalization(z_pre_l), batch_normalization(z_pre_u, m, v))\n",
    "                z += tf.random_normal(tf.shape(z_pre)) * noise_std\n",
    "            else:\n",
    "                # Clean encoder\n",
    "                # batch normalization + update the average mean and variance using batch mean and variance of labeled examples\n",
    "                z = join(update_batch_normalization(z_pre_l, l), batch_normalization(z_pre_u, m, v))\n",
    "            return z\n",
    "\n",
    "        # else:\n",
    "        def eval_batch_norm():\n",
    "            # Evaluation batch normalization\n",
    "            # obtain average mean and variance and use it to normalize the batch\n",
    "            mean = ewma.average(running_mean[l-1])\n",
    "            var = ewma.average(running_var[l-1])\n",
    "            z = batch_normalization(z_pre, mean, var)\n",
    "            # Instead of the above statement, the use of the following 2 statements containing a typo\n",
    "            # consistently produces a 0.2% higher accuracy for unclear reasons.\n",
    "            # m_l, v_l = tf.nn.moments(z_pre_l, axes=[0])\n",
    "            # z = join(batch_normalization(z_pre_l, m_l, mean, var), batch_normalization(z_pre_u, mean, var))\n",
    "            return z\n",
    "\n",
    "        # perform batch normalization according to value of boolean \"training\" placeholder:\n",
    "        z = tf.cond(training, training_batch_norm, eval_batch_norm)\n",
    "\n",
    "        if l == L:\n",
    "            # use softmax activation in output layer\n",
    "            h = tf.nn.softmax(weights['gamma'][l-1] * (z + weights[\"beta\"][l-1]))\n",
    "        else:\n",
    "            # use ReLU activation in hidden layers\n",
    "            h = tf.nn.relu(z + weights[\"beta\"][l-1])\n",
    "        d['labeled']['z'][l], d['unlabeled']['z'][l] = split_lu(z)\n",
    "        d['unlabeled']['m'][l], d['unlabeled']['v'][l] = m, v  # save mean and variance of unlabeled examples for decoding\n",
    "    d['labeled']['h'][l], d['unlabeled']['h'][l] = split_lu(h)\n",
    "    return h, d\n",
    "\n",
    "def g_gauss(z_c, u, size):\n",
    "    \"gaussian denoising function proposed in the original paper\"\n",
    "    wi = lambda inits, name: tf.Variable(inits * tf.ones([size]), name=name)\n",
    "    a1 = wi(0., 'a1')\n",
    "    a2 = wi(1., 'a2')\n",
    "    a3 = wi(0., 'a3')\n",
    "    a4 = wi(0., 'a4')\n",
    "    a5 = wi(0., 'a5')\n",
    "\n",
    "    a6 = wi(0., 'a6')\n",
    "    a7 = wi(1., 'a7')\n",
    "    a8 = wi(0., 'a8')\n",
    "    a9 = wi(0., 'a9')\n",
    "    a10 = wi(0., 'a10')\n",
    "\n",
    "    mu = a1 * tf.sigmoid(a2 * u + a3) + a4 * u + a5\n",
    "    v = a6 * tf.sigmoid(a7 * u + a8) + a9 * u + a10\n",
    "\n",
    "    z_est = (z_c - mu) * v + mu\n",
    "    return z_est\n",
    "\n",
    "def dense_to_one_hot(labels_dense, num_classes=2):\n",
    "    labels_dense=labels_dense.astype(numpy.int)\n",
    "    labels_one_hot = numpy.zeros((labels_dense.shape[0], num_classes))\n",
    "    labels_one_hot[numpy.arange(num_classes), labels_dense] = 1\n",
    "    return labels_one_hot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-30T10:02:38.062463Z",
     "start_time": "2018-01-30T10:02:35.296113Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train = pd.read_csv(\"../LC_data/norm_180127_train.csv\")\n",
    "unlabel = pd.read_csv(\"../LC_data/norm_180127_UL.csv\")\n",
    "valid = pd.read_csv(\"../LC_data/norm_180127_valid.csv\")\n",
    "test = pd.read_csv(\"../LC_data/norm_180127_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train = train.iloc[:,1:].values\n",
    "test = test.iloc[:,1:].values\n",
    "valid = valid.iloc[:,1:].values\n",
    "unlabel = unlabel.iloc[:,2:].values\n",
    "print(train.shape)\n",
    "print(test.shape)\n",
    "print(valid.shape)\n",
    "print(unlabel.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = train[:,1:]\n",
    "train_y = pd.get_dummies(train[:,0]).values\n",
    "valid_X = valid[:,1:]\n",
    "valid_y = pd.get_dummies(valid[:,0]).values\n",
    "test_X = test[:,1:]\n",
    "test_y = pd.get_dummies(test[:,0]).values\n",
    "print(train_X.shape, train_y.shape)\n",
    "print(valid_X.shape, valid_y.shape)\n",
    "print(test_X.shape, test_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "layer_sizes = [68, 40, 15, 5, 2]\n",
    "\n",
    "L = len(layer_sizes) - 1  # number of layers\n",
    "\n",
    "num_examples = train_X.shape[0]\n",
    "num_epochs = 300\n",
    "\n",
    "starter_learning_rate = 0.02\n",
    "\n",
    "decay_after = 15  # epoch after which to begin learning rate decay\n",
    "\n",
    "batch_size = 100\n",
    "U_batch_size = int(unlabel.shape[0] / (num_examples / batch_size))\n",
    "# num_iter = int((num_examples/batch_size) * num_epochs)  # number of loop iterations\n",
    "\n",
    "inputs = tf.placeholder(tf.float32, shape=(None, layer_sizes[0]))\n",
    "outputs = tf.placeholder(tf.float32)\n",
    "\n",
    "\n",
    "shapes = list(zip(layer_sizes[:-1], layer_sizes[1:]))  # shapes of linear layers\n",
    "\n",
    "weights = {'W': [wi(s, \"W\") for s in shapes],  # Encoder weights\n",
    "           'V': [wi(s[::-1], \"V\") for s in shapes],  # Decoder weights\n",
    "           # batch normalization parameter to shift the normalized value\n",
    "           'beta': [bi(0.0, layer_sizes[l+1], \"beta\") for l in range(L)],\n",
    "           # batch normalization parameter to scale the normalized value\n",
    "           'gamma': [bi(1.0, layer_sizes[l+1], \"beta\") for l in range(L)]}\n",
    "\n",
    "noise_std = 0.01  # scaling factor for noise used in corrupted encoder\n",
    "\n",
    "# hyperparameters that denote the importance of each layer\n",
    "denoising_cost = [1.0, 0.50, 0.10, 0.10, 0.10]\n",
    "\n",
    "join = lambda l, u: tf.concat([l, u], 0)\n",
    "labeled = lambda x: tf.slice(x, [0, 0], [batch_size, -1]) if x is not None else x\n",
    "unlabeled = lambda x: tf.slice(x, [batch_size, 0], [-1, -1]) if x is not None else x\n",
    "split_lu = lambda x: (labeled(x), unlabeled(x))\n",
    "\n",
    "training = tf.placeholder(tf.bool)\n",
    "\n",
    "ewma = tf.train.ExponentialMovingAverage(decay=0.99)  # to calculate the moving averages of mean and variance\n",
    "bn_assigns = []  # this list stores the updates to be made to average mean and variance\n",
    "\n",
    "# average mean and variance of all layers\n",
    "running_mean = [tf.Variable(tf.constant(0.0, shape=[l]), trainable=False) for l in layer_sizes[1:]]\n",
    "running_var = [tf.Variable(tf.constant(1.0, shape=[l]), trainable=False) for l in layer_sizes[1:]]\n",
    "\n",
    "print(\"=== Corrupted Encoder ===\")\n",
    "y_c, corr = encoder(inputs, noise_std)\n",
    "\n",
    "print(\"=== Clean Encoder ===\")\n",
    "y, clean = encoder(inputs, 0.0)  # 0.0 -> do not add noise\n",
    "\n",
    "print(\"=== Decoder ===\")\n",
    "\n",
    "# Decoder\n",
    "z_est = {}\n",
    "d_cost = []  # to store the denoising cost of all layers\n",
    "for l in range(L, -1, -1):\n",
    "    print(\"Layer \", l, \": \", layer_sizes[l+1] if l+1 < len(layer_sizes) else None, \" -> \", layer_sizes[l], \", denoising cost: \", denoising_cost[l])\n",
    "    z, z_c = clean['unlabeled']['z'][l], corr['unlabeled']['z'][l]\n",
    "    m, v = clean['unlabeled']['m'].get(l, 0), clean['unlabeled']['v'].get(l, 1-1e-10)\n",
    "    if l == L:\n",
    "        u = unlabeled(y_c)\n",
    "    else:\n",
    "        u = tf.matmul(z_est[l+1], weights['V'][l])\n",
    "    u = batch_normalization(u)\n",
    "    z_est[l] = g_gauss(z_c, u, layer_sizes[l])\n",
    "    z_est_bn = (z_est[l] - m) / v\n",
    "    # append the cost of this layer to d_cost\n",
    "    d_cost.append((tf.reduce_mean(tf.reduce_sum(tf.square(z_est_bn - z), 1)) / layer_sizes[l]) * denoising_cost[l])\n",
    "\n",
    "# calculate total unsupervised cost by adding the denoising cost of all layers\n",
    "u_cost = tf.add_n(d_cost)\n",
    "\n",
    "y_N = labeled(y_c)\n",
    "cost = -tf.reduce_mean(tf.reduce_sum(outputs*tf.log(y_N), 1))  # supervised cost\n",
    "loss = cost + u_cost  # total cost\n",
    "\n",
    "pred_cost = -tf.reduce_mean(tf.reduce_sum(outputs*tf.log(y), 1))  # cost used for prediction\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(outputs, 1))  # no of correct predictions\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\")) * tf.constant(100.0)\n",
    "\n",
    "learning_rate = tf.Variable(starter_learning_rate, trainable=False)\n",
    "train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "\n",
    "# add the updates of batch normalization statistics to train_step\n",
    "bn_updates = tf.group(*bn_assigns)\n",
    "with tf.control_dependencies([train_step]):\n",
    "    train_step = tf.group(bn_updates)\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "print(\"===  Starting Session ===\")\n",
    "sess = tf.Session()\n",
    "\n",
    "i_iter = 0\n",
    "\n",
    "ckpt = tf.train.get_checkpoint_state(CHECK_POINT_PATH)  # get latest checkpoint (if any)\n",
    "if CONTINUE and ckpt and ckpt.model_checkpoint_path:\n",
    "    # if checkpoint exists, restore the parameters and set epoch_n and i_iter\n",
    "    saver.restore(sess, ckpt.model_checkpoint_path)\n",
    "    n_epoch = int(ckpt.model_checkpoint_path.split('-')[1])\n",
    "    i_iter = (n_epoch+1) * (num_examples/batch_size)\n",
    "    print(\"Restored Epoch \", n_epoch)\n",
    "else:\n",
    "    # no checkpoint exists. create checkpoints directory if it does not exist.\n",
    "    if not os.path.exists(CHECK_POINT_PATH):\n",
    "        os.makedirs(CHECK_POINT_PATH)\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "\n",
    "if not CONTINUE:\n",
    "    if os.path.exists(LOG_FILE_PATH):\n",
    "        os.unlink(LOG_FILE_PATH)\n",
    "    for the_file in os.listdir(CHECK_POINT_PATH):\n",
    "        file_path = os.path.join(CHECK_POINT_PATH, the_file)\n",
    "        if os.path.isfile(file_path):\n",
    "            os.unlink(file_path)\n",
    "    print(\"Delete previous model\")\n",
    "\n",
    "print(\"=== Training ===\")\n",
    "print(\"Initial Accuracy: \", sess.run(accuracy, feed_dict={inputs: valid_X, outputs: valid_y, training: False}), \"%\")\n",
    "\n",
    "batch_example_size = int(num_examples / batch_size)\n",
    "for n_epoch in list(range(1,num_epochs+1)):\n",
    "    for n_batch in tqdm(list(range(batch_example_size))):\n",
    "        bat_X, bat_y = train_X[(n_batch*batch_size):((n_batch+1)*batch_size),:], train_y[(n_batch*batch_size):((n_batch+1)*batch_size),:]\n",
    "        bat_X = numpy.vstack((bat_X,unlabel[(n_batch*U_batch_size):((n_batch+1)*U_batch_size),:]))\n",
    "        sess.run(train_step, feed_dict={inputs: bat_X, outputs: bat_y, training: True})\n",
    "    \n",
    "    if n_epoch > decay_after:\n",
    "        # decay learning rate\n",
    "        # learning_rate = starter_learning_rate * ((num_epochs - epoch_n) / (num_epochs - decay_after))\n",
    "        ratio = 1.0 * (num_epochs - (n_epoch+1))  # epoch_n + 1 because learning rate is set for next epoch\n",
    "        ratio = max(0, ratio / (num_epochs - decay_after))\n",
    "        sess.run(learning_rate.assign(starter_learning_rate * ratio))\n",
    "    saver.save(sess, '%s/model.ckpt' % (CHECK_POINT_PATH), n_epoch)\n",
    "    print(\"Epoch \", n_epoch, \", Accuracy: \", sess.run(accuracy, feed_dict={inputs: valid_X, outputs:valid_y, training: False}), \"%\")\n",
    "    with open(LOG_FILE_PATH, 'a+') as train_log:\n",
    "        # write test accu_racy to file \"train_log\"\n",
    "        train_log_w = csv.writer(train_log)\n",
    "        log_i = [n_epoch] + sess.run([accuracy], feed_dict={inputs: valid_X, outputs:valid_y, training: False})\n",
    "        train_log_w.writerow(log_i)\n",
    "\n",
    "print(\"Final Accuracy: \", sess.run(accuracy, feed_dict={inputs: test_X, outputs:test_y, training: False}), \"%\")\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()\n",
    "\n",
    "print(\"===  Starting Session ===\")\n",
    "sess = tf.Session()\n",
    "\n",
    "i_iter = 0\n",
    "ckpt = tf.train.get_checkpoint_state(CHECK_POINT_PATH)  # get latest checkpoint (if any)\n",
    "# if checkpoint exists, restore the parameters and set epoch_n and i_iter\n",
    "saver.restore(sess, ckpt.model_checkpoint_path)\n",
    "n_epoch = int(ckpt.model_checkpoint_path.split('-')[1])\n",
    "i_iter = (n_epoch+1) * (num_examples/batch_size)\n",
    "print(\"Restored Epoch \", n_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Final Accuracy: \", sess.run(accuracy, feed_dict={inputs: test_X, outputs:test_y, training: False}), \"%\")\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "676px",
    "left": "1308.17px",
    "right": "20px",
    "top": "124.994px",
    "width": "350px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}