classifier_rot90.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "import numpy as np\n",
    "import scipy \n",
    "import tensorflow as tf\n",
    "\n",
    "import imageio\n",
    "import gzip\n",
    "from PIL import Image\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.colors as colors\n",
    "import matplotlib.cm as cmx\n",
    "import matplotlib.pyplot as plt\n",
    "#from tqdm import trange, tqdm\n",
    "\n",
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout\n",
    "from keras.optimizers import SGD, Adam, RMSprop, Optimizer\n",
    "from keras.callbacks import Callback\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from helpers import utils\n",
    "# from helpers import protocols\n",
    "# from helpers.keras_utils import LossHistory\n",
    "# from helpers.optimizers import KOOptimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"\"\"\"\"PARAMETERS\"\"\"\"\"\" \n",
    "\n",
    "# Data params\n",
    "input_dim = 784\n",
    "output_dim = 10\n",
    "\n",
    "# Network params\n",
    "n_hidden_units = 2000\n",
    "activation_fn = tf.nn.relu\n",
    "\n",
    "# Optimization params\n",
    "batch_size = 256\n",
    "epochs_per_task = 20 \n",
    "learning_rate=1e-3\n",
    "xi = 0.1\n",
    "\n",
    "# Reset optimizer after each age\n",
    "reset_optimizer = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 3s 0us/step\n"
     ]
    }
   ],
   "source": [
    "# \"\"\"\"\"\"\"CONSTRUCT DATASETS\"\"\"\"\"\" \n",
    "\n",
    "n_tasks = 10\n",
    "full_datasets, final_test_datasets = utils.construct_permute_mnist(num_tasks=n_tasks)\n",
    "# training_datasets, validation_datasets = utils.mk_training_validation_splits(full_datasets, split_fractions=(0.9, 0.1))\n",
    "training_datasets = full_datasets\n",
    "validation_datasets = final_test_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quadratic_regularizer(weights, vars, norm=2):\n",
    "    reg = 0.0\n",
    "    for w in weights:\n",
    "        reg += tf.reduce_sum(vars['omega'][w] * (w - vars['cweights'][w])**norm)\n",
    "    return reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KOOptimizer(Optimizer):\n",
    "    \"\"\"An optimizer whose loss depends on its own updates.\"\"\"\n",
    "\n",
    "    def _allocate_var(self, name=None):\n",
    "        return {w: K.zeros(w.get_shape(), name=name) for w in self.weights}\n",
    "\n",
    "    def _allocate_vars(self, names):\n",
    "        #TODO: add names, better shape/init checking\n",
    "        self.vars = {name: self._allocate_var(name=name) for name in names}\n",
    "\n",
    "    def __init__(self, opt, step_updates=[], task_updates=[], init_updates=[], task_metrics = {}, regularizer_fn=quadratic_regularizer,\n",
    "                lam=1.0, model=None, compute_average_loss=False, compute_average_weights=False, **kwargs):\n",
    "        super(KOOptimizer, self).__init__(**kwargs)\n",
    "        if not isinstance(opt, keras.optimizers.Optimizer):\n",
    "            raise ValueError(\"opt must be an instance of keras.optimizers.Optimizer but got %s\"%type(opt))\n",
    "        if not isinstance(step_updates, OrderedDict):\n",
    "            step_updates = OrderedDict(step_updates)\n",
    "        if not isinstance(task_updates, OrderedDict): task_updates = OrderedDict(task_updates)\n",
    "        if not isinstance(init_updates, OrderedDict): init_updates = OrderedDict(init_updates)\n",
    "        # task_metrics\n",
    "        self.names = set().union(step_updates.keys(), task_updates.keys(), task_metrics.keys())\n",
    "        if 'grads' in self.names or 'deltas' in self.names:\n",
    "            raise ValueError(\"Optimization variables cannot be named 'grads' or 'deltas'\")\n",
    "        self.step_updates = step_updates\n",
    "        self.task_updates = task_updates\n",
    "        self.init_updates = init_updates\n",
    "        self.compute_average_loss = compute_average_loss\n",
    "        self.regularizer_fn = regularizer_fn\n",
    "        # Compute loss and gradients\n",
    "        self.lam = K.variable(value=lam, dtype=tf.float32, name=\"lam\")\n",
    "        self.nb_data = K.variable(value=1.0, dtype=tf.float32, name=\"nb_data\")\n",
    "        self.opt = opt\n",
    "        #self.compute_fisher = compute_fisher\n",
    "        #if compute_fisher and model is None:\n",
    "        #    raise ValueError(\"To compute Fisher information, you need to pass in a Keras model object \")\n",
    "        self.model = model\n",
    "        self.task_metrics = task_metrics\n",
    "        self.compute_average_weights = compute_average_weights\n",
    "\n",
    "    def set_strength(self, val):\n",
    "        K.set_value(self.lam, val)\n",
    "\n",
    "    def set_nb_data(self, nb):\n",
    "        K.set_value(self.nb_data, nb)\n",
    "\n",
    "    def get_updates(self, weights, constraints, initial_loss, model=None):\n",
    "        self.weights = weights\n",
    "        # Allocate variables\n",
    "        with tf.variable_scope(\"KOOptimizer\"):\n",
    "            self._allocate_vars(self.names)\n",
    "\n",
    "        #grads = self.get_gradients(loss, params)\n",
    "\n",
    "        # Compute loss and gradients\n",
    "        self.regularizer = 0.0 if self.regularizer_fn is None else self.regularizer_fn(weights, self.vars)\n",
    "        self.initial_loss = initial_loss\n",
    "        self.loss = initial_loss + self.lam * self.regularizer\n",
    "        with tf.variable_scope(\"wrapped_optimizer\"):\n",
    "            self._weight_update_op, self._grads, self._deltas = compute_updates(self.opt, self.loss, weights)\n",
    "\n",
    "        wrapped_opt_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, \"wrapped_optimizer\")\n",
    "        self.init_opt_vars = tf.variables_initializer(wrapped_opt_vars)\n",
    "\n",
    "        self.vars['unreg_grads'] = dict(zip(weights, tf.gradients(self.initial_loss, weights)))\n",
    "        # Compute updates\n",
    "        self.vars['grads'] = dict(zip(weights, self._grads))\n",
    "        self.vars['deltas'] = dict(zip(weights, self._deltas))\n",
    "        # Keep a pointer to self in vars so we can use it in the updates\n",
    "        self.vars['oopt'] = self\n",
    "        # Keep number of data samples handy for normalization purposes\n",
    "        self.vars['nb_data'] = self.nb_data\n",
    "\n",
    "        if self.compute_average_weights:\n",
    "            with tf.variable_scope(\"weight_emga\") as scope:\n",
    "                weight_ema = tf.train.ExponentialMovingAverage(decay=0.99, zero_debias=True)\n",
    "                self.maintain_weight_averages_op = weight_ema.apply(self.weights)\n",
    "                self.vars['average_weights'] = {w: weight_ema.average(w) for w in self.weights}\n",
    "            self.weight_ema_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope.name)\n",
    "            self.init_weight_ema_vars = tf.variables_initializer(self.weight_ema_vars)\n",
    "            print(\">>>>>\")\n",
    "            K.get_session().run(self.init_weight_ema_vars)\n",
    "        if self.compute_average_loss:\n",
    "            with tf.variable_scope(\"ema\") as scope:\n",
    "                ema = tf.train.ExponentialMovingAverage(decay=0.99, zero_debias=True)\n",
    "                self.maintain_averages_op = ema.apply([self.initial_loss])\n",
    "                self.ema_loss = ema.average(self.initial_loss)\n",
    "                self.prev_loss = tf.Variable(0.0, trainable=False, name=\"prev_loss\")\n",
    "                self.delta_loss = tf.Variable(0.0, trainable=False, name=\"delta_loss\")\n",
    "            self.ema_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope.name)\n",
    "            self.init_ema_vars = tf.variables_initializer(self.ema_vars)\n",
    "#        if self.compute_fisher:\n",
    "#            self._fishers, _, _, _ = compute_fishers(self.model)\n",
    "#            #fishers = compute_fisher_information(model)\n",
    "#            self.vars['fishers'] = dict(zip(weights, self._fishers))\n",
    "#            #fishers, avg_fishers, update_fishers, zero_fishers = compute_fisher_information(model)\n",
    "\n",
    "        def _var_update(vars, update_fn):\n",
    "            updates = []\n",
    "            for w in weights:\n",
    "                updates.append(tf.assign(vars[w], update_fn(self.vars, w, vars[w])))\n",
    "            return tf.group(*updates)\n",
    "\n",
    "        def _compute_vars_update_op(updates):\n",
    "            # Force task updates to happen sequentially\n",
    "            update_op = tf.no_op()\n",
    "            for name, update_fn in updates.items():\n",
    "                with tf.control_dependencies([update_op]):\n",
    "                    update_op = _var_update(self.vars[name], update_fn)\n",
    "            return update_op\n",
    "\n",
    "        self._vars_step_update_op = _compute_vars_update_op(self.step_updates)\n",
    "        self._vars_task_update_op = _compute_vars_update_op(self.task_updates)\n",
    "        self._vars_init_update_op = _compute_vars_update_op(self.init_updates)\n",
    "\n",
    "        # Create task-relevant update ops\n",
    "        reset_ops = []\n",
    "        update_ops = []\n",
    "        for name, metric_fn in self.task_metrics.items():\n",
    "            metric = metric_fn(self)\n",
    "            for w in weights:\n",
    "                reset_ops.append(tf.assign(self.vars[name][w], 0*self.vars[name][w]))\n",
    "                update_ops.append(tf.assign_add(self.vars[name][w], metric[w]))\n",
    "        self._reset_task_metrics_op = tf.group(*reset_ops)\n",
    "        self._update_task_metrics_op = tf.group(*update_ops)\n",
    "\n",
    "        # Each step we update the weights using the optimizer as well as the step-specific variables\n",
    "        self.step_op = tf.group(self._weight_update_op, self._vars_step_update_op)\n",
    "        self.updates.append(self.step_op)\n",
    "        # After each task, run task-specific variable updates\n",
    "        self.task_op = self._vars_task_update_op\n",
    "        self.init_op = self._vars_init_update_op\n",
    "\n",
    "        if self.compute_average_weights:\n",
    "            self.updates.append(self.maintain_weight_averages_op)\n",
    "\n",
    "        if self.compute_average_loss:\n",
    "            self.update_loss_op = tf.assign(self.prev_loss, self.ema_loss)\n",
    "            bupdates = self.updates\n",
    "            with tf.control_dependencies(bupdates + [self.update_loss_op]):\n",
    "                self.updates = [tf.group(*[self.maintain_averages_op])]\n",
    "            self.delta_loss = self.prev_loss - self.ema_loss\n",
    "\n",
    "        return self.updates#[self._base_updates\n",
    "\n",
    "    def init_task_vars(self):\n",
    "        K.get_session().run([self.init_op])\n",
    "\n",
    "    def init_acc_vars(self):\n",
    "        K.get_session().run(self.init_ema_vars)\n",
    "\n",
    "    def init_loss(self, X, y, batch_size):\n",
    "        pass\n",
    "        #sess = K.get_session()\n",
    "        #xi, yi, sample_weights = self.model.model._standardize_user_data(X[:batch_size], y[:batch_size], batch_size=batch_size)\n",
    "        #sess.run(tf.assign(self.prev_loss, self.initial_loss), {self.model.input:xi[0], self.model.model.targets[0]:yi[0], self.model.model.sample_weights[0]:sample_weights[0], K.learning_phase():1})\n",
    "\n",
    "    def update_task_vars(self):\n",
    "        K.get_session().run(self.task_op)\n",
    "\n",
    "    def update_task_metrics(self, X, y, batch_size):\n",
    "        # Reset metric accumulators\n",
    "        n_batch = len(X) // batch_size\n",
    "\n",
    "        sess = K.get_session()\n",
    "        sess.run(self._reset_task_metrics_op)\n",
    "        for i in range(n_batch):\n",
    "            xi, yi, sample_weights = self.model.model._standardize_user_data(X[i * batch_size:(i+1) * batch_size], y[i*batch_size:(i+1)*batch_size], batch_size=batch_size)\n",
    "            sess.run(self._update_task_metrics_op, {self.model.input:xi[0], self.model.model.targets[0]:yi[0], self.model.model.sample_weights[0]:sample_weights[0]})\n",
    "\n",
    "\n",
    "    def reset_optimizer(self):\n",
    "        \"\"\"Reset the optimizer variables\"\"\"\n",
    "        K.get_session().run(self.init_opt_vars)\n",
    "\n",
    "    def get_config(self):\n",
    "        raise ValueError(\"Write the get_config bro\")\n",
    "\n",
    "    def get_numvals_list(self, key='omega'):\n",
    "        \"\"\" Returns list of numerical values such as for instance omegas in reproducible order \"\"\"\n",
    "        variables = self.vars[key]\n",
    "        numvals = []\n",
    "        for p in self.weights:\n",
    "            numval = K.get_value(tf.reshape(variables[p],(-1,)))\n",
    "            numvals.append(numval)\n",
    "        return numvals\n",
    "\n",
    "    def get_numvals(self, key='omega'):\n",
    "        \"\"\" Returns concatenated list of numerical values such as for instance omegas in reproducible order \"\"\"\n",
    "        conc = np.concatenate(self.get_numvals_list(key))\n",
    "        return conc\n",
    "\n",
    "    def get_state(self):\n",
    "        state = []\n",
    "        vs = self.vars\n",
    "        for key in vs.keys():\n",
    "            if key=='oopt': continue\n",
    "            v = vs[key]\n",
    "            for p in v.values():\n",
    "                state.append(K.get_value(p)) # FIXME WhyTF does this not work?\n",
    "        return state\n",
    "\n",
    "    def set_state(self, state):\n",
    "        c = 0\n",
    "        vs = self.vars\n",
    "        for key in vs.keys():\n",
    "            if key=='oopt': continue\n",
    "            v = vs[key]\n",
    "            for p in v.values():\n",
    "                K.set_value(p,state[c])\n",
    "                c += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"\"\"\"\"CONSTRUCT NETWORK, LOSS, UPDATES\"\"\"\"\"\" \n",
    "tf.reset_default_graph()\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.InteractiveSession(config=config)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(n_hidden_units, activation=activation_fn, input_dim=input_dim))\n",
    "model.add(Dense(n_hidden_units, activation=activation_fn))\n",
    "model.add(Dense(output_dim, activation='softmax'))\n",
    "\n",
    "protocol_name, protocol = PATH_INT_PROTOCOL(omega_decay='sum', xi=xi) # protocol_name, protocol = protocols.FISHER_PROTOCOL(omega_decay='sum')\n",
    "opt = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999) # opt = SGD(lr=learning_rate)\n",
    "opt_name = 'adam' # opt_name = 'sgd'\n",
    "oopt = KOOptimizer(opt, model=model, **protocol)\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=oopt, metrics=['accuracy'])\n",
    "\n",
    "history = LossHistory()\n",
    "callbacks = [history]\n",
    "\n",
    "\n",
    "file_prefix = \"data_%s_opt%s_lr%.2e_bs%i_ep%i_tsks%i\"%(protocol_name, opt_name, learning_rate, batch_size, epochs_per_task, n_tasks)\n",
    "datafile_name = \"%s.pkl.gz\"%(file_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"\"\"\"\"TRAIN\"\"\"\"\"\" \n",
    "diag_vals = dict()\n",
    "all_evals = dict()\n",
    "\n",
    "def run_fits(cvals, training_data, valid_data, eval_on_train_set=False):\n",
    "    for cidx, cval_ in enumerate(cvals):\n",
    "        fs = []\n",
    "        evals = []\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        cstuffs = []\n",
    "        cval = cval_\n",
    "        print( \"setting cval\")\n",
    "        oopt.set_strength(cval)\n",
    "        print(\"cval is %e\"%sess.run(oopt.lam))\n",
    "        for age, tidx in enumerate(range(n_tasks)):\n",
    "            print(\"Age %i, cval is=%f\"%(age,cval))\n",
    "            oopt.set_nb_data(len(training_data[tidx][0]))\n",
    "            stuffs = model.fit(training_data[tidx][0], training_data[tidx][1], batch_size, epochs_per_task, callbacks=callbacks,\n",
    "                              verbose=0)\n",
    "            oopt.update_task_metrics(training_data[tidx][0], training_data[tidx][1], batch_size)\n",
    "            oopt.update_task_vars()\n",
    "            ftask = []\n",
    "            for j in range(n_tasks):\n",
    "                if eval_on_train_set:\n",
    "                    f_ = model.evaluate(training_data[j][0], training_data[j][1], batch_size, verbose=0)\n",
    "                else:\n",
    "                    f_ = model.evaluate(valid_data[j][0], valid_data[j][1], batch_size, verbose=0)\n",
    "                ftask.append(np.mean(f_[1]))\n",
    "            evals.append(ftask)\n",
    "            cstuffs.append(stuffs)\n",
    "            \n",
    "            # Re-initialize optimizer variables\n",
    "            if reset_optimizer:\n",
    "                oopt.reset_optimizer()\n",
    "\n",
    "        # diag_vals[cval_] = oopt.get_numvals('omega')\n",
    "        evals = np.array(evals)\n",
    "        all_evals[cval_] = evals\n",
    "        \n",
    "        # backup all_evals to disk\n",
    "        utils.save_zipped_pickle(all_evals, datafile_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0.01, 0.1, 1.0]\n"
     ]
    }
   ],
   "source": [
    "cvals = [0, 0.01, 0.1, 1.0]\n",
    "print(cvals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "setting cval\n",
      "cval is 0.000000e+00\n",
      "Age 0, cval is=0.000000\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "get_updates() got an unexpected keyword argument 'params'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-65-c45724f14ec2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrun_fits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcvals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_datasets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_datasets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-63-dce7addbe5b7>\u001b[0m in \u001b[0;36mrun_fits\u001b[0;34m(cvals, training_data, valid_data, eval_on_train_set)\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0moopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_nb_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtidx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m             stuffs = model.fit(training_data[tidx][0], training_data[tidx][1], batch_size, epochs_per_task, callbacks=callbacks,\n\u001b[0;32m---> 19\u001b[0;31m                               verbose=0)\n\u001b[0m\u001b[1;32m     20\u001b[0m             \u001b[0moopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_task_metrics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtidx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtidx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m             \u001b[0moopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_task_vars\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m    961\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    962\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 963\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m    964\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    965\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1680\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1681\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1682\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1683\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1684\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_make_train_function\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    988\u001b[0m                     training_updates = self.optimizer.get_updates(\n\u001b[1;32m    989\u001b[0m                         \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_collected_trainable_weights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 990\u001b[0;31m                         loss=self.total_loss)\n\u001b[0m\u001b[1;32m    991\u001b[0m                 \u001b[0mupdates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdates\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mtraining_updates\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics_updates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    992\u001b[0m                 \u001b[0;31m# Gets loss and metrics. Updates weights at each call.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: get_updates() got an unexpected keyword argument 'params'"
     ]
    }
   ],
   "source": [
    "run_fits(cvals, training_datasets, validation_datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# backup all_evals to disk\n",
    "# all_evals = dict() # uncomment to delete on disk\n",
    "utils.save_zipped_pickle(all_evals, datafile_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "zero-size array to reduction operation maximum which has no identity",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-71-8a586a09843d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mcmap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_cmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cool'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mcNorm\u001b[0m  \u001b[0;34m=\u001b[0m \u001b[0mcolors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNormalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvmax\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_evals\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m# cNorm  = colors.Normalize()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mscalarMap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcmx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mScalarMappable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcNorm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmap\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcmap\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscalarMap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_clim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36mamax\u001b[0;34m(a, axis, out, keepdims)\u001b[0m\n\u001b[1;32m   2318\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2319\u001b[0m     return _methods._amax(a, axis=axis,\n\u001b[0;32m-> 2320\u001b[0;31m                           out=out, **kwargs)\n\u001b[0m\u001b[1;32m   2321\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2322\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py\u001b[0m in \u001b[0;36m_amax\u001b[0;34m(a, axis, out, keepdims)\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;31m# small reductions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_amax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mumr_maximum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_amin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: zero-size array to reduction operation maximum which has no identity"
     ]
    }
   ],
   "source": [
    "cmap = plt.get_cmap('cool') \n",
    "cNorm  = colors.Normalize(vmin=-4, vmax=np.log(np.max(list(all_evals.keys()))))\n",
    "# cNorm  = colors.Normalize()\n",
    "scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=cmap)\n",
    "print(scalarMap.get_clim())\n",
    "\n",
    "figure(figsize=(14, 4))\n",
    "axs = [subplot(1,n_tasks+1,1)]#, None, None]\n",
    "for i in range(1, n_tasks + 1):\n",
    "    axs.append(subplot(1, n_tasks+1, i+1, sharex=axs[0], sharey=axs[0]))\n",
    "    \n",
    "keys = list(all_evals.keys())\n",
    "sorted_keys = np.sort(keys)\n",
    "\n",
    "\n",
    "for cval in sorted_keys:\n",
    "    evals = all_evals[cval]\n",
    "    for j in range(n_tasks):\n",
    "        colorVal = scalarMap.to_rgba(np.log(cval))\n",
    "        axs[j].plot(evals[:, j], c=colorVal)#, label=\"t%d, c%g\"%(j, cval))\n",
    "    label = \"c=%g\"%cval\n",
    "    average = evals.mean(1)\n",
    "    axs[-1].plot(average, c=colorVal, label=label)\n",
    "    \n",
    "for i, ax in enumerate(axs):\n",
    "    ax.legend(bbox_to_anchor=(1.0,1.0))\n",
    "    ax.set_title((['task %d'%j for j in range(n_tasks)] + ['average'])[i])\n",
    "gcf().tight_layout()\n",
    "\n",
    "\n",
    "for cval in sorted_keys:\n",
    "    stuff = []\n",
    "    for i in range(len(all_evals[cval])):#n_tasks):\n",
    "        stuff.append(all_evals[cval][i][:i+1].mean())\n",
    "    plot(range(1,n_tasks+1), stuff, 'o-', label=\"c=%g\"%cval)\n",
    "axhline(all_evals[cval][0][0], linestyle='--', color='k')\n",
    "    \n",
    "xlabel('Number of tasks')\n",
    "ylabel('Fraction correct')\n",
    "legend(loc='best')\n",
    "ylim(0.9, 1.02)\n",
    "xlim(0.5, 10.5)\n",
    "grid('on')\n",
    "\n",
    "savefig(\"%s.pdf\"%(file_prefix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract labels from MNIST labels into vector\n",
    "def extract_labels(filename, num_images):\n",
    "  with gzip.open(filename) as bytestream:\n",
    "    bytestream.read(8)\n",
    "    buf = bytestream.read(1 * num_images)\n",
    "    labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)\n",
    "  return labels\n",
    "\n",
    "train_labels = extract_labels(\"MNIST-data/train-labels-idx1-ubyte.gz\", 60000)\n",
    "eval_labels = extract_labels(\"MNIST-data/t10k-labels-idx1-ubyte.gz\", 10000)\n",
    "print(np.shape(train_labels))\n",
    "print(np.shape(eval_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# original train\n",
    "train_original = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_original = [\"MNIST-processed-training/original/original{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_original)):\n",
    "    img = np.array(Image.open(images_original[i]))\n",
    "    train_original[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_original))\n",
    "\n",
    "# original test\n",
    "eval_original = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_original = [\"MNIST-processed-test/original/test-original{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_original)):\n",
    "    img = np.array(Image.open(images2_original[i]))\n",
    "    eval_original[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_original))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ROTATE 90 train\n",
    "train_rot90 = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_rot90 = [\"MNIST-processed-training/rot90/rot90{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_rot90)):\n",
    "    img = np.array(Image.open(images_rot90[i]))\n",
    "    train_rot90[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_rot90))\n",
    "\n",
    "# ROTATE 90 test\n",
    "eval_rot90 = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_rot90 = [\"MNIST-processed-test/rot90/test-rot90{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_rot90)):\n",
    "    img = np.array(Image.open(images2_rot90[i]))\n",
    "    eval_rot90[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_rot90))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# checkerboard train\n",
    "train_checkerboard = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_checkerboard = [\"MNIST-processed-training/checkerboard/fullcheck{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_checkerboard)):\n",
    "    img = np.array(Image.open(images_checkerboard[i]))\n",
    "    train_checkerboard[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_checkerboard))\n",
    "\n",
    "# checkerboard test\n",
    "eval_checkerboard = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_checkerboard = [\"MNIST-processed-test/checkerboard/test-checkerboard{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_checkerboard)):\n",
    "    img = np.array(Image.open(images2_checkerboard[i]))\n",
    "    eval_checkerboard[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_checkerboard))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# INV train\n",
    "train_inv = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_inv = [\"MNIST-processed-training/Inv/inv{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_inv)):\n",
    "    img = np.array(Image.open(images_inv[i]))\n",
    "    train_inv[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_inv))\n",
    "\n",
    "# INV test\n",
    "eval_inv = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_inv = [\"MNIST-processed-test/inv/test-inv{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_inv)):\n",
    "    img = np.array(Image.open(images2_inv[i]))\n",
    "    eval_inv[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_inv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# invbot train\n",
    "train_invbot = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_invbot = [\"MNIST-processed-training/invbot/invbot{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_invbot)):\n",
    "    img = np.array(Image.open(images_invbot[i]))\n",
    "    train_invbot[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_invbot))\n",
    "\n",
    "# invbot test\n",
    "eval_invbot = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_invbot = [\"MNIST-processed-test/invbot/test-invbot{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_invbot)):\n",
    "    img = np.array(Image.open(images2_invbot[i]))\n",
    "    eval_invbot[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_invbot))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fliplr train\n",
    "train_fliplr = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_fliplr = [\"MNIST-processed-training/fliplr/fliplr{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_fliplr)):\n",
    "    img = np.array(Image.open(images_fliplr[i]))\n",
    "    train_fliplr[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_fliplr))\n",
    "\n",
    "# fliplr test\n",
    "eval_fliplr = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_fliplr = [\"MNIST-processed-test/fliplr/test-fliplr{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_fliplr)):\n",
    "    img = np.array(Image.open(images2_fliplr[i]))\n",
    "    eval_fliplr[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_fliplr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cutud train\n",
    "train_cutud = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_cutud = [\"MNIST-processed-training/cutud/cutUD{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_cutud)):\n",
    "    img = np.array(Image.open(images_cutud[i]))\n",
    "    train_cutud[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_cutud))\n",
    "\n",
    "# cutud test\n",
    "eval_cutud = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_cutud = [\"MNIST-processed-test/cutud/test-cutud{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_cutud)):\n",
    "    img = np.array(Image.open(images2_cutud[i]))\n",
    "    eval_cutud[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_cutud))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# flipud train\n",
    "train_flipud = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_flipud = [\"MNIST-processed-training/flipud/flipud{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_flipud)):\n",
    "    img = np.array(Image.open(images_flipud[i]))\n",
    "    train_flipud[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_data))\n",
    "\n",
    "# flipud test\n",
    "eval_flipud = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_flipud = [\"MNIST-processed-test/flipud/test-flipud{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_flipud)):\n",
    "    img = np.array(Image.open(images2_flipud[i]))\n",
    "    eval_flipud[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_flipud))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cnn_model_fn(features, labels, mode):\n",
    "\n",
    "    # Input Layer\n",
    "    input_layer = tf.reshape(features[\"x\"], [-1, 28, 28, 1])\n",
    "\n",
    "    # Convolutional layer 1\n",
    "    conv1 = tf.layers.conv2d(\n",
    "        inputs = input_layer,\n",
    "        filters = 32,\n",
    "        kernel_size=[5,5],\n",
    "        padding=\"same\",\n",
    "        activation=tf.nn.relu\n",
    "    )\n",
    "\n",
    "    # Pooling 1\n",
    "    pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)\n",
    "\n",
    "    # Convolutional layer 2 and pooling layer\n",
    "    conv2 = tf.layers.conv2d(\n",
    "        inputs=pool1,\n",
    "        filters=64,\n",
    "        kernel_size=[5,5],\n",
    "        padding=\"same\",\n",
    "        activation=tf.nn.relu\n",
    "    )\n",
    "\n",
    "    # Pooling 2 with flattening\n",
    "    pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2,2], strides=2)\n",
    "    pool2_flat=tf.reshape(pool2, [-1, 7 * 7 * 64])\n",
    "\n",
    "    # Dense layer with dropout \n",
    "    dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)\n",
    "    dropout=tf.layers.dropout(inputs=dense, rate=0.4, training=mode==tf.estimator.ModeKeys.TRAIN)\n",
    "    logits = tf.layers.dense(inputs=dropout, units=10)\n",
    "\n",
    "    # Generate predictions\n",
    "    predictions = {\n",
    "        \"classes\": tf.argmax(input=logits, axis=1),\n",
    "        \"probabilities\": tf.nn.softmax(logits, name=\"softmax_tensor\")\n",
    "    }\n",
    "\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)\n",
    "\n",
    "    # Caluclate loss\n",
    "    loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n",
    "\n",
    "    # Configure training op\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\n",
    "        train_op = optimizer.minimize(\n",
    "        loss=loss,\n",
    "        global_step=tf.train.get_global_step())\n",
    "        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n",
    "\n",
    "    # Add evaluation metrics\n",
    "    eval_metric_ops = {\n",
    "        \"accuracy\": tf.metrics.accuracy(labels=labels, predictions=predictions[\"classes\"])}\n",
    "\n",
    "    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Estimator\n",
    "mnist_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn, model_dir=\"/tmp/mnist_convnet_model\")\n",
    "\n",
    "# Logging predictions\n",
    "tensors_to_log = {\"probabilities\": \"softmax_tensor\"}\n",
    "logging_hook = tf.train.LoggingTensorHook(\n",
    "    tensors=tensors_to_log, \n",
    "    every_n_iter=50)\n",
    "\n",
    "# Our application logic will be added here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Load rotated images for training\n",
    "# images = [\"MNIST-processed-training/rot90/rot90{0}.png\".format(k) for k in range(1,60001)]\n",
    "\n",
    "# train_data = [] \n",
    "# for image in images: \n",
    "#     img = Image.open(image)\n",
    "#     arr = np.array(img)\n",
    "#     train_data.append(arr)\n",
    "# #     train_data.append(imageio.imread(image))\n",
    "    \n",
    "# # Flatten images    \n",
    "# for i in range(len(train_data)):\n",
    "#     train_data[i] = np.array(train_data[i].flatten())\n",
    "    \n",
    "# print(np.shape(train_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mnist = tf.contrib.learn.datasets.load_dataset(\"mnist\")\n",
    "\n",
    "# train_data_int = mnist.train.images\n",
    "# train_labels_int = np.asarray(mnist.train.labels, dtype=np.int32)\n",
    "# eval_data_int = mnist.test.images\n",
    "# eval_labels_int = np.asarray(mnist.test.labels, dtype=np.int32)\n",
    "\n",
    "# print(np.shape(train_data_int))\n",
    "# print(np.shape(train_labels_int))\n",
    "# print(np.shape(eval_data_int))\n",
    "# print(np.shape(eval_labels_int))\n",
    "\n",
    "# def parse_img(filename, label):\n",
    "#     img_str = tf.read_file(filename)\n",
    "#     img_dec = tf.image.decode_image(img_str)\n",
    "# #     img_res = tf.image.resize_images(img_dec, [28, 28])\n",
    "#     return img_dec, label\n",
    "\n",
    "# # Load rotated images for training\n",
    "# filenames = tf.constant([\"MNIST-processed-training/rot90/rot90{0}.png\".format(k) for k in range(1,60001)])\n",
    "\n",
    "# labels = train_labels\n",
    "# dataset = tf.data.Dataset.from_tensor_slices((filenames, labels))\n",
    "# dataset = dataset.map(parse_img)\n",
    "    \n",
    "# print(np.shape(dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(unused_argv):\n",
    "    # Training on original\n",
    "    train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x={\"x\":train_original},\n",
    "        y=train_labels,\n",
    "        batch_size=1000,\n",
    "        num_epochs=None,\n",
    "        shuffle=True)\n",
    "    \n",
    "    mnist_classifier.train(\n",
    "        input_fn=train_input_fn,\n",
    "        steps=200,\n",
    "        hooks=[logging_hook])\n",
    "\n",
    "    # Evaluation on original\n",
    "    eval_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x={\"x\": eval_original},\n",
    "        y=eval_labels,\n",
    "        num_epochs=1,\n",
    "        shuffle=False)\n",
    "    \n",
    "    eval_results=mnist_classifier.evaluate(input_fn=eval_input_fn)\n",
    "    print(eval_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "  tf.app.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(unused_argv):\n",
    "    # Training on rot90\n",
    "    train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x={\"x\":train_rot90},\n",
    "        y=train_labels,\n",
    "        batch_size=1000,\n",
    "        num_epochs=None,\n",
    "        shuffle=True)\n",
    "    \n",
    "    mnist_classifier.train(\n",
    "        input_fn=train_input_fn,\n",
    "        steps=200,\n",
    "        hooks=[logging_hook])\n",
    "\n",
    "    # Evaluation on rot90\n",
    "    eval_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x={\"x\": eval_rot90},\n",
    "        y=eval_labels,\n",
    "        num_epochs=1,\n",
    "        shuffle=False)\n",
    "    \n",
    "    eval_results=mnist_classifier.evaluate(input_fn=eval_input_fn)\n",
    "    print(eval_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "  tf.app.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def main(unused_argv):\n",
    "#     # Training on rot90\n",
    "#     train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "#         x={\"x\":train_inv},\n",
    "#         y=train_labels,\n",
    "#         batch_size=1000,\n",
    "#         num_epochs=None,\n",
    "#         shuffle=True)\n",
    "    \n",
    "#     mnist_classifier.train(\n",
    "#         input_fn=train_input_fn,\n",
    "#         steps=200,\n",
    "#         hooks=[logging_hook])\n",
    "\n",
    "#     # Evaluation on rot90\n",
    "#     eval_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "#         x={\"x\": eval_inv},\n",
    "#         y=eval_labels,\n",
    "#         num_epochs=1,\n",
    "#         shuffle=False)\n",
    "    \n",
    "#     eval_results=mnist_classifier.evaluate(input_fn=eval_input_fn)\n",
    "#     print(eval_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if __name__ == \"__main__\":\n",
    "#   tf.app.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model.layers[-1].get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CONSTANTS\n",
    "batch_size = 128\n",
    "num_classes = 10\n",
    "epochs = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout\n",
    "from keras.optimizers import SGD, Adam, RMSprop\n",
    "from keras.callbacks import Callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from pathint import protocols\n",
    "#from pathint.optimizers import KOOptimizer\n",
    "#from pathint.keras_utils import LossHistory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = keras.utils.to_categorical(train_original, num_classes)\n",
    "y_test = keras.utils.to_categorical(test_original, num_classes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_original = train_original.reshape(60000, 784)\n",
    "x_eval_original = eval_original.reshape(10000, 784)\n",
    "\n",
    "x_train_original = reshape_train_original.astype('float32')\n",
    "x_eval_original = reshape_eval_original.astype('float32')\n",
    "\n",
    "x_train_original /= 255\n",
    "x_eval_original /= 255\n",
    "\n",
    "print(reshape_train_original.shape[0], 'train samples')\n",
    "print(reshape_eval_original.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(train_labels, num_classes)\n",
    "y_eval = keras.utils.to_categorical(eval_labels, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(512, activation='relu', input_shape=(784,)))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(512, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train_original, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=epochs,\n",
    "                    verbose=1,\n",
    "                    validation_data=(x_eval_original, y_eval))\n",
    "score = model.evaluate(x_eval_original, y_eval, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = model.layers[-1].get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.shape(weights[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}