RTE Playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import theano\n",
    "import nnb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from __future__ import print_function\n",
    "import codecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from chios import feats_glove\n",
    "\n",
    "glove = feats_glove._get_glove_dict('data/glove.6B.50d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import sys\n",
    "\n",
    "def glove_bow(sent, ndim=2):\n",
    "    tokens = word_tokenize(sent)\n",
    "    gtokens = [glove[t] for t in tokens if t in glove]\n",
    "    if not gtokens:\n",
    "        return np.zeros((1, 50)) if ndim == 2 else np.zeros(50)\n",
    "    gtokens = np.array(gtokens)\n",
    "    if ndim == 2:\n",
    "        return gtokens\n",
    "    else:\n",
    "        return gtokens.mean(axis=0)\n",
    "\n",
    "def load_sents(fname, ndim=2):\n",
    "    labels = []\n",
    "    sent0 = []\n",
    "    sent1 = []\n",
    "    sent2 = []\n",
    "    i = 0\n",
    "    print('', file=sys.stderr)\n",
    "    with open(fname) as f:\n",
    "        c = csv.DictReader(f)\n",
    "        for l in c:\n",
    "            labels.append(int(l['isCorrect']))\n",
    "            sent0.append(l['qText'].decode('utf8'))\n",
    "            sent1.append(l['aText'].decode('utf8'))\n",
    "            sent2.append(l['abstract'].decode('utf8'))\n",
    "            print('%d\\r', i, end='', file=sys.stderr)\n",
    "            i += 1\n",
    "    print('', file=sys.stderr)\n",
    "    return (np.array([glove_bow(s, ndim) for s in sent0]),\n",
    "            np.array([glove_bow(s, ndim) for s in sent1]),\n",
    "            np.array([glove_bow(s, ndim) for s in sent2]),\n",
    "            np.array(labels, dtype='int32'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 4825\n",
      "\n",
      " 4825\n"
     ]
    }
   ],
   "source": [
    "s0, s1, s2, y = load_sents('solraoccur-ck12.csv')\n",
    "s0a, s1a, s2a, y = load_sents('solraoccur-ck12.csv', ndim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1261, 4826)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.size(y[y==1]), np.size(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.51564,  0.19855,  0.21705,  0.09245,  0.14504,  0.33042,\n",
       "         0.21705,  0.11008, -0.14578]),\n",
       " array([ 0.18672222,  0.25739411, -0.50672167, -0.13433971,  0.57635244,\n",
       "         0.41791667, -0.38240133, -0.31375211, -0.00928411,  0.05760128,\n",
       "        -0.05973359,  0.17729222, -0.08681472,  0.00613817,  0.43688222,\n",
       "        -0.10499633, -0.04779122,  0.21209866, -0.4491907 , -0.25576667,\n",
       "        -0.12617167,  0.24664222,  0.08513444,  0.17559189,  0.31300111,\n",
       "        -1.73918222, -0.66073689,  0.07377767,  0.45624778, -0.14804556,\n",
       "         2.859     , -0.296494  , -0.36126567, -0.46518333, -0.01673733,\n",
       "         0.03278956, -0.02011344,  0.060729  ,  0.17624778, -0.13548533,\n",
       "        -0.05123767,  0.12163253, -0.185221  ,  0.43272011,  0.29413433,\n",
       "        -0.11844867,  0.13815767, -0.12095689,  0.23342541,  0.43198778]))"
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s0[0][:,0], s0a[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def balance_dataset(tdata, y):\n",
    "    tdata = list(tdata)\n",
    "    class1 = np.where(y==1)[0]\n",
    "    n_imbal = np.sum(y==-1) - np.sum(y==1)\n",
    "    for i in np.random.choice(class1, size=n_imbal):\n",
    "        tdata.append(tdata[i])\n",
    "    return tdata\n",
    "\n",
    "def try_model(model, s0, s1, s2, y, val_on_train=False, lrate=0.01, L2_reg=0.0):\n",
    "    print('Preprocessing...')\n",
    "    y = y*2-1  # classify as -1/+1 for tanh activation    \n",
    "    \n",
    "    n = s0.shape[0]\n",
    "    tdata = [[s0[i], s1[i], s2[i], np.array([y[i]], dtype='int32')] for i in range(n)]\n",
    "    n_train = int(n*0.8) if not val_on_train else 0    \n",
    "    if n_train > 0:\n",
    "        tdata_train = balance_dataset(tdata[:n_train], y[:n_train])\n",
    "        tdata_val = balance_dataset(tdata[n_train:], y[n_train:])\n",
    "    else:\n",
    "        n_train = n\n",
    "        tdata_train = balance_dataset(tdata, y)\n",
    "        tdata_val = balance_dataset(tdata, y)\n",
    "    \n",
    "    print('Cost function...')\n",
    "    expected_output = nnb.InputLayer(ndim=1, dtype='int32')\n",
    "    cost_func = nnb.cost.MeanSquareError()\n",
    "    network_cost = (model & expected_output) | cost_func\n",
    "    \n",
    "    print('Compiling...')\n",
    "    trainer = nnb.train.SGDTrainer(model=network_cost, learning_rate=lrate, L2_reg=L2_reg)\n",
    "    print('Training...')\n",
    "\n",
    "    train_sup = nnb.train.TrainSupervisor(trainer=trainer, dataset=list(tdata_train), eval_dataset=list(tdata_val),\n",
    "                                          batch_size=n_train//100 if n_train >= 100 else n_train,\n",
    "                                          epochs_num=200, max_no_improve=40)\n",
    "    train_sup.train()\n",
    "\n",
    "    print('Checking...')\n",
    "    ff = model.compile()\n",
    "    for i in range(7):\n",
    "        print(i, ff(tdata_train[i][0], tdata_train[i][1], tdata_train[i][2]), tdata_train[i][-1])\n",
    "    def eval_dataset(ff, tdata):\n",
    "        return np.array([ff(tdata[i][0], tdata[i][1], tdata[i][2])[0] for i in range(len(tdata))])\n",
    "    def accuracy(tdata, yy):\n",
    "        y = np.array([t[-1][0] for t in tdata])\n",
    "        n_cor = np.sum(y[yy > 0]+1)/2 + np.sum(-y[yy < 0]+1)/2\n",
    "        return n_cor / float(np.shape(yy)[0])\n",
    "    print('Accuracy:',\n",
    "          'train', accuracy(tdata_train, eval_dataset(ff, tdata_train)),\n",
    "          'val', accuracy(tdata_val, eval_dataset(ff, tdata_val)),\n",
    "          'val_base', accuracy(tdata_val, np.zeros(np.shape(tdata_val)[0])-1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bag-of-words Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 0.00549978415171 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501362623832\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 0.00539129972458 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503881584326\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 0.0053907473882 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497145278045\n",
      "New best!\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 0.00527220169703 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495405360828\n",
      "New best!\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 0.00519735018412 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494150887001\n",
      "New best!\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 0.00517598390579 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49417934948\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 0.00519626537959 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494296162773\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 0.00520528554916 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495565837841\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 0.0051739970843 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498853705701\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 0.00517669916153 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498582239545\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 0.00519020160039 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498576160236\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 0.00525878270467 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497302176748\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 0.00540873209635 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49381808675\n",
      "New best!\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 0.0051917831103 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515861089803\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 0.00522441466649 minutes.\n",
      "Evaluating...\n",
      "Error = 0.493692968119\n",
      "New best!\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 0.00520411729813 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509553328505\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 0.00525681575139 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499896159862\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 0.00521080096563 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502951860082\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 0.00522662003835 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497142872134\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 0.00522808631261 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496862204815\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 0.0053240497907 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495135953209\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 0.00523244937261 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498971385132\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 0.00529078245163 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498201341127\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 0.00534888108571 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504442302592\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 0.00529346466064 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495927930815\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 0.00518519878387 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49772955467\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 0.00523625214895 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498852659371\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 0.00522571802139 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494939422793\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 0.00525558392207 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494522617358\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 0.00531095266342 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504766566958\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 0.00526976982752 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494608166328\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 0.00528730154037 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496587571244\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 0.00523878335953 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496137797241\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 0.00531571706136 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499799528409\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 0.00530395110448 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496189293578\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 0.00519131819407 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496786677362\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 0.00529083410899 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495096530459\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 0.00531123479207 minutes.\n",
      "Evaluating...\n",
      "Error = 0.4968953071\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 0.00534201463064 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494447355729\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 0.0054208834966 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494739498645\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 0.00551505088806 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498369757923\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 0.00525753498077 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498978957103\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 0.00528826713562 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498713773553\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 0.00532711744308 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49752885637\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 0.00544883410136 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500351958295\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 0.00538436571757 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503797997244\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 0.00540731747945 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497872127372\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 0.00534911553065 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494708510758\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 0.00533268451691 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494905336957\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 0.0053013364474 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498153025572\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 0.00539723237356 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495895548788\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 0.00538180271784 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49597391698\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 0.00561686754227 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495946780709\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 0.00538660287857 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508830522601\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 0.00536410013835 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497012046124\n",
      "Finished!\n",
      "Best error: 0.493692968119\n",
      "Checking...\n",
      "0 [ 0.10245764] [-1]\n",
      "1 [ 0.24511647] [1]\n",
      "2 [ 0.06273516] [-1]\n",
      "3 [-0.07669717] [-1]\n",
      "4 [ 0.09157349] [-1]\n",
      "5 [ 0.04323913] [-1]\n",
      "6 [ 0.14705638] [1]\n",
      "Accuracy: train 0.566024518389 val 0.549295774648 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# the simplest model that just tries to classify by linear combination of averages\n",
    "L0 = nnb.InputLayer(ndim=1)\n",
    "L1 = nnb.InputLayer(ndim=1)\n",
    "L2 = nnb.InputLayer(ndim=1)\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=50*3, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model, s0a, s1a, s2a, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 568,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 0.016385948658 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50049292534\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 0.0174004832904 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50610278228\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 0.0171181996663 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506186629279\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 0.0171550989151 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50113928957\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 0.016674331824 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500789331228\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 0.0174194494883 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501884327586\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 0.0167300343513 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50029072991\n",
      "New best!\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 0.0172538638115 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501607817685\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 0.0180797855059 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49996798107\n",
      "New best!\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 0.0168595989545 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49991514989\n",
      "New best!\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 0.0171553174655 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506292575768\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 0.0167421658834 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50297907638\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 0.0168383161227 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503881286942\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 0.0180901845296 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499415426634\n",
      "New best!\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 0.0174206177394 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499495360896\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 0.016449602445 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502178917424\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 0.0172587196032 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504706076476\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 0.0166318496068 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509952831313\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 0.113059234619 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505266438083\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 0.0182055830956 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499374131947\n",
      "New best!\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 0.0184769034386 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499520022169\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 0.0167285362879 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502826417367\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 0.0171048482259 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499715092818\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 0.0170586665471 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503091470741\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 0.0175805648168 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499994169642\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 0.0180044809977 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499713480451\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 0.0162137349447 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502316740728\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 0.0173502326012 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502023047951\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 0.0158873836199 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507041055449\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 0.018289633592 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499704849719\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 0.0170264840126 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504158655807\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 0.0162334839503 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506872135489\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 0.0163709322611 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501245463753\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 0.0173902193705 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504146195408\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 0.0152581532796 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500532388868\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 0.01839158535 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499649344978\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 0.0170514663061 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500174021362\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 0.0179256836573 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503627732455\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 0.0167336861293 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499751723006\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 0.0157539129257 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509885304596\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 0.0158346494039 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506895792505\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 0.0170587817828 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504338473275\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 0.0182851632436 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499510435116\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 0.11200884978 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501784787628\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 0.0172656496366 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500092201381\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 0.0178255995115 minutes.\n",
      "Evaluating...\n",
      "Error = 0.526037818468\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 0.0177946011225 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506461093483\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 0.0183282176654 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501223775826\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 0.0168531815211 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503064908468\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 0.0173494338989 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501523538517\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 0.015671702226 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49972793703\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 0.0154407183329 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500616995069\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 0.0165719509125 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500017235791\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 0.0162053664525 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499381351953\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 0.0172932664553 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501904416361\n",
      "~Epoch 56~\n",
      "[--------- ] \n",
      "Finished. Took 0.0170731306076 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502884411693\n",
      "~Epoch 57~\n",
      "[--------- ] \n",
      "Finished. Took 0.0168791333834 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501796720756\n",
      "~Epoch 58~\n",
      "[--------- ] \n",
      "Finished. Took 0.0171793818474 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499603203131\n",
      "~Epoch 59~\n",
      "[--------- ] \n",
      "Finished. Took 0.0159686843554 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504181407303\n",
      "~Epoch 60~\n",
      "[--------- ] \n",
      "Finished. Took 0.0154766003291 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50160360474\n",
      "Finished!\n",
      "Best error: 0.499374131947\n",
      "Checking...\n",
      "0 [ 0.01719661] [-1]\n",
      "1 [ 0.02384478] [1]\n",
      "2 [-0.00468837] [-1]\n",
      "3 [-0.00032959] [-1]\n",
      "4 [-0.01221382] [-1]\n",
      "5 [-0.01094366] [-1]\n",
      "6 [-0.01057806] [1]\n",
      "Accuracy: train 0.532224168126 val 0.514084507042 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# only slightly more complex model that first merges l0, l1, then tries to learn a linear combination of that with l2\n",
    "L0 = nnb.InputLayer(ndim=1)\n",
    "L1 = nnb.InputLayer(ndim=1)\n",
    "C01 = nnb.ConcatenationModel()\n",
    "C1 = nnb.PerceptronLayer(insize=2*50, outsize=50)\n",
    "qajoin = (L0 & L1) | C01 | C1\n",
    "L2 = nnb.InputLayer(ndim=1)\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=2*50, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (qajoin & L2) | CC | C2\n",
    "try_model(model, s0a, s1a, s2a, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 577,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217814326286 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500357608689\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217702666918 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500007342751\n",
      "New best!\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 0.021881600221 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500145769065\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218525330226 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500343043051\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218734184901 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500018127542\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218481183052 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500029008664\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223319689433 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501230559823\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218985676765 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500803193592\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218106150627 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500096080358\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218416015307 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500542697909\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218163013458 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503701656726\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218620498975 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499996685223\n",
      "New best!\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218279838562 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500114915747\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 0.021864883105 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50000234246\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218339165052 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501091637636\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220268646876 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500141559823\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218552192052 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501337095868\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217992186546 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500326763224\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218602180481 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500073103204\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218064347903 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500221115396\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219269990921 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500677313315\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218431830406 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499999411133\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218532681465 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500826726601\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217517852783 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500886390623\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218867341677 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500462269425\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217462658882 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501476886195\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223957339923 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500028026309\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 0.021883781751 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500362921846\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218784968058 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500223828294\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222222487132 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502864350179\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218279361725 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500619307896\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218044996262 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499987406644\n",
      "New best!\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221875667572 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500366175275\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219057679176 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500410528936\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218756159147 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500048850828\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217797160149 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500113452477\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218860348066 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500028451843\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219083189964 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499986161081\n",
      "New best!\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220331986745 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501674818963\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219795306524 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500021788593\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218236843745 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500132441928\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218075990677 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499985138935\n",
      "New best!\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219311475754 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500253047816\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223954002062 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503078734515\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218530495962 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499995136601\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223164161046 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500738841335\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219388167063 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499974096477\n",
      "New best!\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218135674795 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500120409531\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217928806941 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500184527706\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218890508016 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500130238553\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220821499825 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50109280612\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218738516172 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500248211362\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218495686849 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50014239258\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218565146128 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50086558956\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220512986183 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499970071088\n",
      "New best!\n",
      "~Epoch 56~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218093832334 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500149102842\n",
      "~Epoch 57~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220838824908 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500033025376\n",
      "~Epoch 58~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218762477239 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501198219803\n",
      "~Epoch 59~\n",
      "[--------- ] \n",
      "Finished. Took 0.0230452815692 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50012025529\n",
      "~Epoch 60~\n",
      "[--------- ] \n",
      "Finished. Took 0.021840496858 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500077270832\n",
      "~Epoch 61~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218392491341 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50077927355\n",
      "~Epoch 62~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217747489611 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499984892315\n",
      "~Epoch 63~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217939813932 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499997648061\n",
      "~Epoch 64~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217553496361 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500380485899\n",
      "~Epoch 65~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222050984701 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499986363764\n",
      "~Epoch 66~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219154993693 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499977693214\n",
      "~Epoch 67~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222742160161 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50068828969\n",
      "~Epoch 68~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221931139628 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499993818277\n",
      "~Epoch 69~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223215699196 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500456752386\n",
      "~Epoch 70~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218123356501 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501516249963\n",
      "~Epoch 71~\n",
      "[--------- ] \n",
      "Finished. Took 0.022450085481 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500085789755\n",
      "~Epoch 72~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222800334295 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500401278801\n",
      "~Epoch 73~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218959013621 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50036435486\n",
      "~Epoch 74~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217820684115 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502541472291\n",
      "~Epoch 75~\n",
      "[--------- ] \n",
      "Finished. Took 0.0224252978961 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499963902512\n",
      "New best!\n",
      "~Epoch 76~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217240651449 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500101250946\n",
      "~Epoch 77~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217859864235 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50040999318\n",
      "~Epoch 78~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217722177505 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500510304056\n",
      "~Epoch 79~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217543999354 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500318563833\n",
      "~Epoch 80~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223845322927 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499998462164\n",
      "~Epoch 81~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218147158623 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502026415632\n",
      "~Epoch 82~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222399671872 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502093156353\n",
      "~Epoch 83~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217820326487 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500024211545\n",
      "~Epoch 84~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218458970388 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502331249226\n",
      "~Epoch 85~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217954158783 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499950856299\n",
      "New best!\n",
      "~Epoch 86~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223637660344 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500109465447\n",
      "~Epoch 87~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217625339826 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499951075783\n",
      "~Epoch 88~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221950531006 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499984625191\n",
      "~Epoch 89~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217854340871 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499943631552\n",
      "New best!\n",
      "~Epoch 90~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223787148794 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500429406791\n",
      "~Epoch 91~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217892169952 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499942773631\n",
      "New best!\n",
      "~Epoch 92~\n",
      "[--------- ] \n",
      "Finished. Took 0.0224108338356 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500466631022\n",
      "~Epoch 93~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217609643936 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500321091069\n",
      "~Epoch 94~\n",
      "[--------- ] \n",
      "Finished. Took 0.0224106987317 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500188895129\n",
      "~Epoch 95~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217641671499 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501258692385\n",
      "~Epoch 96~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223159313202 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500267478797\n",
      "~Epoch 97~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217929840088 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500833885566\n",
      "~Epoch 98~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219456831614 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500383458707\n",
      "~Epoch 99~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218558192253 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500447765059\n",
      "~Epoch 100~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219075163205 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500164301496\n",
      "~Epoch 101~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217956503232 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500216465245\n",
      "~Epoch 102~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220236817996 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499932070404\n",
      "New best!\n",
      "~Epoch 103~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217332323392 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500514964888\n",
      "~Epoch 104~\n",
      "[--------- ] \n",
      "Finished. Took 0.022004433473 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500646231343\n",
      "~Epoch 105~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222724517186 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500148437788\n",
      "~Epoch 106~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221728165944 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500274829908\n",
      "~Epoch 107~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221629142761 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500846483374\n",
      "~Epoch 108~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219543178876 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501688023001\n",
      "~Epoch 109~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220215161641 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499926322139\n",
      "New best!\n",
      "~Epoch 110~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222080667814 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50008128092\n",
      "~Epoch 111~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218286991119 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500781777389\n",
      "~Epoch 112~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221534848213 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50023895332\n",
      "~Epoch 113~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218267679214 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500057674475\n",
      "~Epoch 114~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218214670817 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500339113468\n",
      "~Epoch 115~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217881321907 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500115919768\n",
      "~Epoch 116~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221712311109 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500773938284\n",
      "~Epoch 117~\n",
      "[--------- ] \n",
      "Finished. Took 0.021998099486 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499916980547\n",
      "New best!\n",
      "~Epoch 118~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217414657275 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500236518758\n",
      "~Epoch 119~\n",
      "[--------- ] \n",
      "Finished. Took 0.0216476639112 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501275343322\n",
      "~Epoch 120~\n",
      "[--------- ] \n",
      "Finished. Took 0.0216891646385 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500820501055\n",
      "~Epoch 121~\n",
      "[--------- ] \n",
      "Finished. Took 0.021661400795 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499966690951\n",
      "~Epoch 122~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222122470538 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500708797497\n",
      "~Epoch 123~\n",
      "[--------- ] \n",
      "Finished. Took 0.0216812650363 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499953660078\n",
      "~Epoch 124~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217606663704 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499911319753\n",
      "New best!\n",
      "~Epoch 125~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218466838201 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499930339418\n",
      "~Epoch 126~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220993836721 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501150152719\n",
      "~Epoch 127~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218068679174 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500139854953\n",
      "~Epoch 128~\n",
      "[--------- ] \n",
      "Finished. Took 0.021918686231 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500142661993\n",
      "~Epoch 129~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218550006549 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49991073876\n",
      "New best!\n",
      "~Epoch 130~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221742153168 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500179002928\n",
      "~Epoch 131~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217227856318 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50179531013\n",
      "~Epoch 132~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219247659047 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500713304135\n",
      "~Epoch 133~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218584696452 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499900592966\n",
      "New best!\n",
      "~Epoch 134~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218326489131 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501571134841\n",
      "~Epoch 135~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217429637909 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500682062652\n",
      "~Epoch 136~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218285004298 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499940946878\n",
      "~Epoch 137~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218677163124 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500320651113\n",
      "~Epoch 138~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218434810638 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499903030568\n",
      "~Epoch 139~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219102342923 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499947078422\n",
      "~Epoch 140~\n",
      "[--------- ] \n",
      "Finished. Took 0.021824836731 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499934056377\n",
      "~Epoch 141~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220553994179 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501482778227\n",
      "~Epoch 142~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219018022219 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499918216915\n",
      "~Epoch 143~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221315145493 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499931968784\n",
      "~Epoch 144~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219853838285 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50260866734\n",
      "~Epoch 145~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217618703842 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499908084401\n",
      "~Epoch 146~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217352986336 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501762268779\n",
      "~Epoch 147~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217153668404 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499888432519\n",
      "New best!\n",
      "~Epoch 148~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217912157377 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49991514925\n",
      "~Epoch 149~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221135814985 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500508044643\n",
      "~Epoch 150~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217555681864 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499878423691\n",
      "New best!\n",
      "~Epoch 151~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217488646507 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501063758992\n",
      "~Epoch 152~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219927668571 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500039604084\n",
      "~Epoch 153~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223458488782 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500237799145\n",
      "~Epoch 154~\n",
      "[--------- ] \n",
      "Finished. Took 0.021802063783 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499974968699\n",
      "~Epoch 155~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217899004618 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499871304411\n",
      "New best!\n",
      "~Epoch 156~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222612182299 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500234555797\n",
      "~Epoch 157~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217857321103 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499989003734\n",
      "~Epoch 158~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217375675837 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500779556462\n",
      "~Epoch 159~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217485666275 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500264661635\n",
      "~Epoch 160~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220564325651 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49995073226\n",
      "~Epoch 161~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217218160629 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50009481034\n",
      "~Epoch 162~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223880847295 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49990221258\n",
      "~Epoch 163~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219489177068 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499990916639\n",
      "~Epoch 164~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217112660408 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49985870774\n",
      "New best!\n",
      "~Epoch 165~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219016313553 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502086411787\n",
      "~Epoch 166~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220451315244 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49993755507\n",
      "~Epoch 167~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217249512672 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50059977622\n",
      "~Epoch 168~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222033818563 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500236614824\n",
      "~Epoch 169~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217567841212 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499930695424\n",
      "~Epoch 170~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217022657394 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499936242315\n",
      "~Epoch 171~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220777829488 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499943283797\n",
      "~Epoch 172~\n",
      "[--------- ] \n",
      "Finished. Took 0.021776398023 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499881586553\n",
      "~Epoch 173~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220500866572 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499841521847\n",
      "New best!\n",
      "~Epoch 174~\n",
      "[--------- ] \n",
      "Finished. Took 0.021843679746 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500319624653\n",
      "~Epoch 175~\n",
      "[--------- ] \n",
      "Finished. Took 0.0219239989916 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499876892588\n",
      "~Epoch 176~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221277674039 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499884155379\n",
      "~Epoch 177~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221422672272 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50003074753\n",
      "~Epoch 178~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217188676198 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500425126917\n",
      "~Epoch 179~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217609683673 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500124382613\n",
      "~Epoch 180~\n",
      "[--------- ] \n",
      "Finished. Took 0.0216934164365 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499967479468\n",
      "~Epoch 181~\n",
      "[--------- ] \n",
      "Finished. Took 0.0221087495486 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499938273837\n",
      "~Epoch 182~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218360026677 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499844550209\n",
      "~Epoch 183~\n",
      "[--------- ] \n",
      "Finished. Took 0.0222194512685 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499820791011\n",
      "New best!\n",
      "~Epoch 184~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217803200086 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500965870433\n",
      "~Epoch 185~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223155140877 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499817250333\n",
      "New best!\n",
      "~Epoch 186~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218931317329 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500598134078\n",
      "~Epoch 187~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218052188555 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500309503611\n",
      "~Epoch 188~\n",
      "[--------- ] \n",
      "Finished. Took 0.0216904004415 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499840548108\n",
      "~Epoch 189~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218406001727 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50047814393\n",
      "~Epoch 190~\n",
      "[--------- ] \n",
      "Finished. Took 0.0216904481252 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500518066015\n",
      "~Epoch 191~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217192490896 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500123707496\n",
      "~Epoch 192~\n",
      "[--------- ] \n",
      "Finished. Took 0.0223292509715 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499802596093\n",
      "New best!\n",
      "~Epoch 193~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217316865921 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500608947229\n",
      "~Epoch 194~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218078176181 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499852098886\n",
      "~Epoch 195~\n",
      "[--------- ] \n",
      "Finished. Took 0.021713300546 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499825558896\n",
      "~Epoch 196~\n",
      "[--------- ] \n",
      "Finished. Took 0.0220940828323 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50015228768\n",
      "~Epoch 197~\n",
      "[--------- ] \n",
      "Finished. Took 0.0218266487122 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499791890101\n",
      "New best!\n",
      "~Epoch 198~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217582146327 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499838253324\n",
      "~Epoch 199~\n",
      "[--------- ] \n",
      "Finished. Took 0.0216853817304 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500524829256\n",
      "~Epoch 200~\n",
      "[--------- ] \n",
      "Finished. Took 0.0217870473862 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499790650898\n",
      "New best!\n",
      "Finished!\n",
      "Best error: 0.499790650898\n",
      "Checking...\n",
      "0 [ 0.00058826] [-1]\n",
      "1 [ 0.00492114] [1]\n",
      "2 [-0.00190044] [-1]\n",
      "3 [-0.00193494] [-1]\n",
      "4 [-0.00560141] [-1]\n",
      "5 [-0.00566186] [-1]\n",
      "6 [-0.00692472] [1]\n",
      "Accuracy: train 0.526970227671 val 0.519014084507 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# extension of the above with extra hidden layer\n",
    "L0 = nnb.InputLayer(ndim=1)\n",
    "L1 = nnb.InputLayer(ndim=1)\n",
    "C01 = nnb.ConcatenationModel()\n",
    "C1a = nnb.PerceptronLayer(insize=2*50, outsize=50)\n",
    "C1b = nnb.PerceptronLayer(insize=50, outsize=25, activation_func=nnb.activation.tanh)\n",
    "qajoin = (L0 & L1) | C01 | C1a | C1b\n",
    "L2 = nnb.InputLayer(ndim=1)\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2a = nnb.PerceptronLayer(insize=50+25, outsize=25)\n",
    "C2b = nnb.PerceptronLayer(insize=25, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (qajoin & L2) | CC | C2a | C2b\n",
    "try_model(model, s0a, s1a, s2a, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/NNBlocks-0.1.a.dev-py2.7.egg/nnb/utils/options.py:87: RuntimeWarning: Trying to set a nonexistent option activation_func. The option will be added.\n",
      "/usr/local/lib/python2.7/dist-packages/Theano-0.7.0-py2.7.egg/theano/scan_module/scan_perform_ext.py:133: RuntimeWarning: numpy.ndarray size changed, may indicate binary incompatibility\n",
      "  from scan_perform.scan_perform import *\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[----------] \n",
      "Finished. Took 0.0628002643585 minutes.\n",
      "Evaluating...\n",
      "Error = 0.579013365775\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[----------] \n",
      "Finished. Took 0.0613608002663 minutes.\n",
      "Evaluating...\n",
      "Error = 0.526072313757\n",
      "New best!\n",
      "~Epoch 3~\n",
      "[----------] \n",
      "Finished. Took 0.0614840467771 minutes.\n",
      "Evaluating...\n",
      "Error = 0.599939170187\n",
      "~Epoch 4~\n",
      "[----------] \n",
      "Finished. Took 0.0616254846255 minutes.\n",
      "Evaluating...\n",
      "Error = 0.538122519286\n",
      "~Epoch 5~\n",
      "[----------] \n",
      "Finished. Took 0.0624274810155 minutes.\n",
      "Evaluating...\n",
      "Error = 0.520889916575\n",
      "New best!\n",
      "~Epoch 6~\n",
      "[----------] \n",
      "Finished. Took 0.0617220362027 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508043564146\n",
      "New best!\n",
      "~Epoch 7~\n",
      "[----------] \n",
      "Finished. Took 0.0608848492304 minutes.\n",
      "Evaluating...\n",
      "Error = 0.546115475313\n",
      "~Epoch 8~\n",
      "[----------] \n",
      "Finished. Took 0.0606798330943 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49982176631\n",
      "New best!\n",
      "~Epoch 9~\n",
      "[----------] \n",
      "Finished. Took 0.0618589520454 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500385014691\n",
      "~Epoch 10~\n",
      "[----------] \n",
      "Finished. Took 0.0616268992424 minutes.\n",
      "Evaluating...\n",
      "Error = 0.510619253362\n",
      "~Epoch 11~\n",
      "[----------] \n",
      "Finished. Took 0.0615065177282 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501457666716\n",
      "~Epoch 12~\n",
      "[----------] \n",
      "Finished. Took 0.0620406190554 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500167136914\n",
      "~Epoch 13~\n",
      "[----------] \n",
      "Finished. Took 0.0615134676298 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500701618263\n",
      "~Epoch 14~\n",
      "[----------] \n",
      "Finished. Took 0.0612983981768 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501649299037\n",
      "~Epoch 15~\n",
      "[----------] \n",
      "Finished. Took 0.061228565375 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503657579582\n",
      "~Epoch 16~\n",
      "[----------] \n",
      "Finished. Took 0.0622821331024 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513317290218\n",
      "~Epoch 17~\n",
      "[----------] \n",
      "Finished. Took 0.0625649174054 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505686460013\n",
      "~Epoch 18~\n",
      "[----------] \n",
      "Finished. Took 0.0618648529053 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508849900466\n",
      "~Epoch 19~\n",
      "[----------] \n",
      "Finished. Took 0.0611614028613 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506720038988\n",
      "~Epoch 20~\n",
      "[----------] \n",
      "Finished. Took 0.0611968159676 minutes.\n",
      "Evaluating...\n",
      "Error = 0.510837683492\n",
      "~Epoch 21~\n",
      "[----------] \n",
      "Finished. Took 0.0612424174945 minutes.\n",
      "Evaluating...\n",
      "Error = 0.519248539606\n",
      "~Epoch 22~\n",
      "[----------] \n",
      "Finished. Took 0.0614284674327 minutes.\n",
      "Evaluating...\n",
      "Error = 0.536801341237\n",
      "~Epoch 23~\n",
      "[----------] \n",
      "Finished. Took 0.0610617677371 minutes.\n",
      "Evaluating...\n",
      "Error = 0.522130556755\n",
      "~Epoch 24~\n",
      "[----------] \n",
      "Finished. Took 0.0615648825963 minutes.\n",
      "Evaluating...\n",
      "Error = 0.514905696492\n",
      "~Epoch 25~\n",
      "[----------] \n",
      "Finished. Took 0.0626250982285 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517563258617\n",
      "~Epoch 26~\n",
      "[----------] \n",
      "Finished. Took 0.06218303442 minutes.\n",
      "Evaluating...\n",
      "Error = 0.522165570119\n",
      "~Epoch 27~\n",
      "[----------] \n",
      "Finished. Took 0.0624438166618 minutes.\n",
      "Evaluating...\n",
      "Error = 0.53138893505\n",
      "~Epoch 28~\n",
      "[----------] \n",
      "Finished. Took 0.061695766449 minutes.\n",
      "Evaluating...\n",
      "Error = 0.564958027853\n",
      "~Epoch 29~\n",
      "[----------] \n",
      "Finished. Took 0.0616178830465 minutes.\n",
      "Evaluating...\n",
      "Error = 0.541820092249\n",
      "~Epoch 30~\n",
      "[----------] \n",
      "Finished. Took 0.0617184003194 minutes.\n",
      "Evaluating...\n",
      "Error = 0.533303030657\n",
      "~Epoch 31~\n",
      "[----------] \n",
      "Finished. Took 0.0621129631996 minutes.\n",
      "Evaluating...\n",
      "Error = 0.567610408792\n",
      "~Epoch 32~\n",
      "[----------] \n",
      "Finished. Took 0.0609832485517 minutes.\n",
      "Evaluating...\n",
      "Error = 0.522701602924\n",
      "~Epoch 33~\n",
      "[----------] \n",
      "Finished. Took 0.0622516830762 minutes.\n",
      "Evaluating...\n",
      "Error = 0.532527038418\n",
      "~Epoch 34~\n",
      "[----------] \n",
      "Finished. Took 0.062995882829 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517965266741\n",
      "~Epoch 35~\n",
      "[----------] \n",
      "Finished. Took 0.0665150682131 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517279568882\n",
      "~Epoch 36~\n",
      "[----------] \n",
      "Finished. Took 0.0639619509379 minutes.\n",
      "Evaluating...\n",
      "Error = 0.521106540607\n",
      "~Epoch 37~\n",
      "[----------] \n",
      "Finished. Took 0.0643773516019 minutes.\n",
      "Evaluating...\n",
      "Error = 0.533636861422\n",
      "~Epoch 38~\n",
      "[----------] \n",
      "Finished. Took 0.0623411814372 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515082270926\n",
      "~Epoch 39~\n",
      "[----------] \n",
      "Finished. Took 0.0630791664124 minutes.\n",
      "Evaluating...\n",
      "Error = 0.51194228559\n",
      "~Epoch 40~\n",
      "[----------] \n",
      "Finished. Took 0.0626374999682 minutes.\n",
      "Evaluating...\n",
      "Error = 0.51186730109\n",
      "~Epoch 41~\n",
      "[----------] \n",
      "Finished. Took 0.0622809330622 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508634403656\n",
      "~Epoch 42~\n",
      "[----------] \n",
      "Finished. Took 0.0634076197942 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506455740075\n",
      "~Epoch 43~\n",
      "[----------] \n",
      "Finished. Took 0.0616853674253 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501513848954\n",
      "~Epoch 44~\n",
      "[----------] \n",
      "Finished. Took 0.0630856156349 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499427387009\n",
      "New best!\n",
      "~Epoch 45~\n",
      "[----------] \n",
      "Finished. Took 0.0619325002035 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50431392091\n",
      "~Epoch 46~\n",
      "[----------] \n",
      "Finished. Took 0.0618892629941 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505535934691\n",
      "~Epoch 47~\n",
      "[----------] \n",
      "Finished. Took 0.0627247174581 minutes.\n",
      "Evaluating...\n",
      "Error = 0.528584584581\n",
      "~Epoch 48~\n",
      "[----------] \n",
      "Finished. Took 0.0613739331563 minutes.\n",
      "Evaluating...\n",
      "Error = 0.492724980553\n",
      "New best!\n",
      "~Epoch 49~\n",
      "[----------] \n",
      "Finished. Took 0.0621403813362 minutes.\n",
      "Evaluating...\n",
      "Error = 0.493532641232\n",
      "~Epoch 50~\n",
      "[----------] \n",
      "Finished. Took 0.0615621169408 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494636810372\n",
      "~Epoch 51~\n",
      "[----------] \n",
      "Finished. Took 0.0609093825022 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501587535502\n",
      "~Epoch 52~\n",
      "[----------] \n",
      "Finished. Took 0.0609608014425 minutes.\n",
      "Evaluating...\n",
      "Error = 0.492460245529\n",
      "New best!\n",
      "~Epoch 53~\n",
      "[----------] \n",
      "Finished. Took 0.0616674502691 minutes.\n",
      "Evaluating...\n",
      "Error = 0.491126309643\n",
      "New best!\n",
      "~Epoch 54~\n",
      "[----------] \n",
      "Finished. Took 0.0609399477641 minutes.\n",
      "Evaluating...\n",
      "Error = 0.536531643311\n",
      "~Epoch 55~\n",
      "[----------] \n",
      "Finished. Took 0.0612622817357 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49433619608\n",
      "~Epoch 56~\n",
      "[----------] \n",
      "Finished. Took 0.0616774002711 minutes.\n",
      "Evaluating...\n",
      "Error = 0.491779455468\n",
      "~Epoch 57~\n",
      "[----------] \n",
      "Finished. Took 0.0614414493243 minutes.\n",
      "Evaluating...\n",
      "Error = 0.531344356585\n",
      "~Epoch 58~\n",
      "[----------] \n",
      "Finished. Took 0.0615602850914 minutes.\n",
      "Evaluating...\n",
      "Error = 0.491127874302\n",
      "~Epoch 59~\n",
      "[----------] \n",
      "Finished. Took 0.0612180352211 minutes.\n",
      "Evaluating...\n",
      "Error = 0.491473329758\n",
      "~Epoch 60~\n",
      "[----------] \n",
      "Finished. Took 0.0624972343445 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497978810145\n",
      "~Epoch 61~\n",
      "[----------] \n",
      "Finished. Took 0.0625387152036 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49407984313\n",
      "~Epoch 62~\n",
      "[----------] \n",
      "Finished. Took 0.0619624813398 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515441338072\n",
      "~Epoch 63~\n",
      "[----------] \n",
      "Finished. Took 0.0615761478742 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49907329093\n",
      "~Epoch 64~\n",
      "[----------] \n",
      "Finished. Took 0.0612406651179 minutes.\n",
      "Evaluating...\n",
      "Error = 0.526570704042\n",
      "~Epoch 65~\n",
      "[----------] \n",
      "Finished. Took 0.0623576998711 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507454831186\n",
      "~Epoch 66~\n",
      "[----------] \n",
      "Finished. Took 0.061577530702 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503178976487\n",
      "~Epoch 67~\n",
      "[----------] \n",
      "Finished. Took 0.0619952162107 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503949618324\n",
      "~Epoch 68~\n",
      "[----------] \n",
      "Finished. Took 0.0612798651059 minutes.\n",
      "Evaluating...\n",
      "Error = 0.550490270244\n",
      "~Epoch 69~\n",
      "[----------] \n",
      "Finished. Took 0.0607627828916 minutes.\n",
      "Evaluating...\n",
      "Error = 0.529199092843\n",
      "~Epoch 70~\n",
      "[----------] \n",
      "Finished. Took 0.0617208480835 minutes.\n",
      "Evaluating...\n",
      "Error = 0.542438848923\n",
      "~Epoch 71~\n",
      "[----------] \n",
      "Finished. Took 0.0621061325073 minutes.\n",
      "Evaluating...\n",
      "Error = 0.519034598408\n",
      "~Epoch 72~\n",
      "[----------] \n",
      "Finished. Took 0.0616780002912 minutes.\n",
      "Evaluating...\n",
      "Error = 0.521686247451\n",
      "~Epoch 73~\n",
      "[----------] \n",
      "Finished. Took 0.0614182313283 minutes.\n",
      "Evaluating...\n",
      "Error = 0.523923966402\n",
      "~Epoch 74~\n",
      "[----------] \n",
      "Finished. Took 0.0622708678246 minutes.\n",
      "Evaluating...\n",
      "Error = 0.518335980756\n",
      "~Epoch 75~\n",
      "[----------] \n",
      "Finished. Took 0.0612687826157 minutes.\n",
      "Evaluating...\n",
      "Error = 0.525362131085\n",
      "~Epoch 76~\n",
      "[----------] \n",
      "Finished. Took 0.0620802164078 minutes.\n",
      "Evaluating...\n",
      "Error = 0.535046949177\n",
      "~Epoch 77~\n",
      "[----------] \n",
      "Finished. Took 0.0618770678838 minutes.\n",
      "Evaluating...\n",
      "Error = 0.534272724504\n",
      "~Epoch 78~\n",
      "[----------] \n",
      "Finished. Took 0.0613665819168 minutes.\n",
      "Evaluating...\n",
      "Error = 0.533117098877\n",
      "~Epoch 79~\n",
      "[----------] \n",
      "Finished. Took 0.0615491827329 minutes.\n",
      "Evaluating...\n",
      "Error = 0.546520015236\n",
      "~Epoch 80~\n",
      "[----------] \n",
      "Finished. Took 0.0622786323229 minutes.\n",
      "Evaluating...\n",
      "Error = 0.551984717525\n",
      "~Epoch 81~\n",
      "[----------] \n",
      "Finished. Took 0.0619322856267 minutes.\n",
      "Evaluating...\n",
      "Error = 0.552090363357\n",
      "~Epoch 82~\n",
      "[----------] \n",
      "Finished. Took 0.061626235644 minutes.\n",
      "Evaluating...\n",
      "Error = 0.5478371882\n",
      "~Epoch 83~\n",
      "[----------] \n",
      "Finished. Took 0.0610935688019 minutes.\n",
      "Evaluating...\n",
      "Error = 0.549961080188\n",
      "~Epoch 84~\n",
      "[----------] \n",
      "Finished. Took 0.0613050818443 minutes.\n",
      "Evaluating...\n",
      "Error = 0.559586452567\n",
      "~Epoch 85~\n",
      "[----------] \n",
      "Finished. Took 0.0616555849711 minutes.\n",
      "Evaluating...\n",
      "Error = 0.553366178116\n",
      "~Epoch 86~\n",
      "[----------] \n",
      "Finished. Took 0.0613197684288 minutes.\n",
      "Evaluating...\n",
      "Error = 0.551822840379\n",
      "~Epoch 87~\n",
      "[----------] \n",
      "Finished. Took 0.0608694473902 minutes.\n",
      "Evaluating...\n",
      "Error = 0.553776539802\n",
      "~Epoch 88~\n",
      "[----------] \n",
      "Finished. Took 0.0614246527354 minutes.\n",
      "Evaluating...\n",
      "Error = 0.55466073987\n",
      "~Epoch 89~\n",
      "[----------] \n",
      "Finished. Took 0.0620656649272 minutes.\n",
      "Evaluating...\n",
      "Error = 0.567885843519\n",
      "~Epoch 90~\n",
      "[----------] \n",
      "Finished. Took 0.0612316330274 minutes.\n",
      "Evaluating...\n",
      "Error = 0.569749925411\n",
      "~Epoch 91~\n",
      "[----------] \n",
      "Finished. Took 0.0613452355067 minutes.\n",
      "Evaluating...\n",
      "Error = 0.573693002971\n",
      "~Epoch 92~\n",
      "[----------] \n",
      "Finished. Took 0.0613347331683 minutes.\n",
      "Evaluating...\n",
      "Error = 0.576835756868\n",
      "~Epoch 93~\n",
      "[----------] \n",
      "Finished. Took 0.0612351020177 minutes.\n",
      "Evaluating...\n",
      "Error = 0.583230478496\n",
      "Finished!\n",
      "Best error: 0.491126309643\n",
      "Checking...\n",
      "0 [-0.96179989] [-1]\n",
      "1 [-0.27628886] [1]\n",
      "2 [ 0.01255821] [-1]\n",
      "3 [-0.65091075] [-1]\n",
      "4 [-0.97002695] [-1]\n",
      "5 [-0.76677572] [-1]\n",
      "6 [ 0.24124381] [1]\n",
      "Accuracy: train 0.775423728814 val 0.516666666667 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# a more complex model that does away with the averaging of words in sentences, instead using an RNN:\n",
    "# [:200] subset for fast sanity check; without intermediate perceptrons\n",
    "# FIRST SUCCESS-LOOKING STUFF!!!\n",
    "\n",
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model, s0[:200], s1[:200], s2[:200], y[:200])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[----------] \n",
      "Finished. Took 0.0582627534866 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508872767739\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[----------] \n",
      "Finished. Took 0.0580453356107 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503538897169\n",
      "New best!\n",
      "~Epoch 3~\n",
      "[----------] \n",
      "Finished. Took 0.0566072503726 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50161568079\n",
      "New best!\n",
      "~Epoch 4~\n",
      "[----------] \n",
      "Finished. Took 0.0579905351003 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500734741892\n",
      "New best!\n",
      "~Epoch 5~\n",
      "[----------] \n",
      "Finished. Took 0.0575432817141 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509149692829\n",
      "~Epoch 6~\n",
      "[----------] \n",
      "Finished. Took 0.0577412327131 minutes.\n",
      "Evaluating...\n",
      "Error = 0.537933883044\n",
      "~Epoch 7~\n",
      "[----------] \n",
      "Finished. Took 0.0573727806409 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501164665474\n",
      "~Epoch 8~\n",
      "[----------] \n",
      "Finished. Took 0.0585675199827 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501627615539\n",
      "~Epoch 9~\n",
      "[----------] \n",
      "Finished. Took 0.0581838488579 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501858241981\n",
      "~Epoch 10~\n",
      "[----------] \n",
      "Finished. Took 0.0578620672226 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502042887982\n",
      "~Epoch 11~\n",
      "[----------] \n",
      "Finished. Took 0.0572620828946 minutes.\n",
      "Evaluating...\n",
      "Error = 0.51156725059\n",
      "~Epoch 12~\n",
      "[----------] \n",
      "Finished. Took 0.058463982741 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508391747759\n",
      "~Epoch 13~\n",
      "[----------] \n",
      "Finished. Took 0.0574010809263 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504913540294\n",
      "~Epoch 14~\n",
      "[----------] \n",
      "Finished. Took 0.058275449276 minutes.\n",
      "Evaluating...\n",
      "Error = 0.52803624384\n",
      "~Epoch 15~\n",
      "[----------] \n",
      "Finished. Took 0.0579015493393 minutes.\n",
      "Evaluating...\n",
      "Error = 0.550780887383\n",
      "~Epoch 16~\n",
      "[----------] \n",
      "Finished. Took 0.0580205480258 minutes.\n",
      "Evaluating...\n",
      "Error = 0.511797721575\n",
      "~Epoch 17~\n",
      "[----------] \n",
      "Finished. Took 0.05717253685 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506709948413\n",
      "~Epoch 18~\n",
      "[----------] \n",
      "Finished. Took 0.0576379179955 minutes.\n",
      "Evaluating...\n",
      "Error = 0.530107546389\n",
      "~Epoch 19~\n",
      "[----------] \n",
      "Finished. Took 0.057345867157 minutes.\n",
      "Evaluating...\n",
      "Error = 0.533157323401\n",
      "~Epoch 20~\n",
      "[----------] \n",
      "Finished. Took 0.0585215846697 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508574052758\n",
      "~Epoch 21~\n",
      "[----------] \n",
      "Finished. Took 0.057880115509 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509893116701\n",
      "~Epoch 22~\n",
      "[----------] \n",
      "Finished. Took 0.0574174483617 minutes.\n",
      "Evaluating...\n",
      "Error = 0.62876921968\n",
      "~Epoch 23~\n",
      "[----------] \n",
      "Finished. Took 0.0571619351705 minutes.\n",
      "Evaluating...\n",
      "Error = 0.519573474993\n",
      "~Epoch 24~\n",
      "[----------] \n",
      "Finished. Took 0.0577168663343 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513798838432\n",
      "~Epoch 25~\n",
      "[----------] \n",
      "Finished. Took 0.0585041364034 minutes.\n",
      "Evaluating...\n",
      "Error = 0.532809099304\n",
      "~Epoch 26~\n",
      "[----------] \n",
      "Finished. Took 0.0575534343719 minutes.\n",
      "Evaluating...\n",
      "Error = 0.518861161191\n",
      "~Epoch 27~\n",
      "[----------] \n",
      "Finished. Took 0.0581449151039 minutes.\n",
      "Evaluating...\n",
      "Error = 0.557539821032\n",
      "~Epoch 28~\n",
      "[----------] \n",
      "Finished. Took 0.0583305835724 minutes.\n",
      "Evaluating...\n",
      "Error = 0.558245399642\n",
      "~Epoch 29~\n",
      "[----------] \n",
      "Finished. Took 0.0584337353706 minutes.\n",
      "Evaluating...\n",
      "Error = 0.540217381179\n",
      "~Epoch 30~\n",
      "[----------] \n",
      "Finished. Took 0.0573572675387 minutes.\n",
      "Evaluating...\n",
      "Error = 0.529778467337\n",
      "~Epoch 31~\n",
      "[----------] \n",
      "Finished. Took 0.0578389326731 minutes.\n",
      "Evaluating...\n",
      "Error = 0.543086308419\n",
      "~Epoch 32~\n",
      "[----------] \n",
      "Finished. Took 0.0580480019252 minutes.\n",
      "Evaluating...\n",
      "Error = 0.534400069062\n",
      "~Epoch 33~\n",
      "[----------] \n",
      "Finished. Took 0.0578056017558 minutes.\n",
      "Evaluating...\n",
      "Error = 0.536052220524\n",
      "~Epoch 34~\n",
      "[----------] \n",
      "Finished. Took 0.058070230484 minutes.\n",
      "Evaluating...\n",
      "Error = 0.559478997077\n",
      "~Epoch 35~\n",
      "[----------] \n",
      "Finished. Took 0.0577588001887 minutes.\n",
      "Evaluating...\n",
      "Error = 0.538730119242\n",
      "~Epoch 36~\n",
      "[----------] \n",
      "Finished. Took 0.0577538172404 minutes.\n",
      "Evaluating...\n",
      "Error = 0.554323835909\n",
      "~Epoch 37~\n",
      "[----------] \n",
      "Finished. Took 0.0576082348824 minutes.\n",
      "Evaluating...\n",
      "Error = 0.540343870427\n",
      "~Epoch 38~\n",
      "[----------] \n",
      "Finished. Took 0.0584973017375 minutes.\n",
      "Evaluating...\n",
      "Error = 0.562448175264\n",
      "~Epoch 39~\n",
      "[----------] \n",
      "Finished. Took 0.0583022673925 minutes.\n",
      "Evaluating...\n",
      "Error = 0.550094330609\n",
      "~Epoch 40~\n",
      "[----------] \n",
      "Finished. Took 0.0587592522303 minutes.\n",
      "Evaluating...\n",
      "Error = 0.547474219925\n",
      "~Epoch 41~\n",
      "[----------] \n",
      "Finished. Took 0.0580926656723 minutes.\n",
      "Evaluating...\n",
      "Error = 0.571531549217\n",
      "~Epoch 42~\n",
      "[----------] \n",
      "Finished. Took 0.058218284448 minutes.\n",
      "Evaluating...\n",
      "Error = 0.551195592155\n",
      "~Epoch 43~\n",
      "[----------] \n",
      "Finished. Took 0.0576384186745 minutes.\n",
      "Evaluating...\n",
      "Error = 0.548841151405\n",
      "~Epoch 44~\n",
      "[----------] \n",
      "Finished. Took 0.0577274839083 minutes.\n",
      "Evaluating...\n",
      "Error = 0.567675919012\n",
      "Finished!\n",
      "Best error: 0.500734741892\n",
      "Checking...\n",
      "0 [-0.0120187] [-1]\n",
      "1 [-0.01491104] [1]\n",
      "2 [-0.01237685] [-1]\n",
      "3 [-0.00836088] [-1]\n",
      "4 [-0.01087497] [-1]\n",
      "5 [-0.01006033] [-1]\n",
      "6 [-0.00993011] [1]\n",
      "Accuracy: train 0.5 val 0.5 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# cautious regularization even better\n",
    "\n",
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model, s0[:200], s1[:200], s2[:200], y[:200], L2_reg=1.0/(3*50+10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[----------] \n",
      "Finished. Took 0.0634068171183 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500880341737\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[----------] \n",
      "Finished. Took 0.0678580164909 minutes.\n",
      "Evaluating...\n",
      "Error = 0.518072418206\n",
      "~Epoch 3~\n",
      "[----------] \n",
      "Finished. Took 0.0635004003843 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507660027651\n",
      "~Epoch 4~\n",
      "[----------] \n",
      "Finished. Took 0.0633414824804 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515954869169\n",
      "~Epoch 5~\n",
      "[----------] \n",
      "Finished. Took 0.0632675806681 minutes.\n",
      "Evaluating...\n",
      "Error = 0.552496332767\n",
      "~Epoch 6~\n",
      "[----------] \n",
      "Finished. Took 0.0627001841863 minutes.\n",
      "Evaluating...\n",
      "Error = 0.526260543693\n",
      "~Epoch 7~\n",
      "[----------] \n",
      "Finished. Took 0.0622161189715 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501870471261\n",
      "~Epoch 8~\n",
      "[----------] \n",
      "Finished. Took 0.0637435317039 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502330481714\n",
      "~Epoch 9~\n",
      "[----------] \n",
      "Finished. Took 0.0628800829252 minutes.\n",
      "Evaluating...\n",
      "Error = 0.535322910578\n",
      "~Epoch 10~\n",
      "[----------] \n",
      "Finished. Took 0.0644400000572 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502044442783\n",
      "~Epoch 11~\n",
      "[----------] \n",
      "Finished. Took 0.0638324181239 minutes.\n",
      "Evaluating...\n",
      "Error = 0.566909464981\n",
      "~Epoch 12~\n",
      "[----------] \n",
      "Finished. Took 0.0624670346578 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505899666744\n",
      "~Epoch 13~\n",
      "[----------] \n",
      "Finished. Took 0.0629362662633 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499444281084\n",
      "New best!\n",
      "~Epoch 14~\n",
      "[----------] \n",
      "Finished. Took 0.0663687467575 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502427704446\n",
      "~Epoch 15~\n",
      "[----------] \n",
      "Finished. Took 0.062012664477 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517459931504\n",
      "~Epoch 16~\n",
      "[----------] \n",
      "Finished. Took 0.0633774161339 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502631599485\n",
      "~Epoch 17~\n",
      "[----------] \n",
      "Finished. Took 0.0628344337145 minutes.\n",
      "Evaluating...\n",
      "Error = 0.510605165683\n",
      "~Epoch 18~\n",
      "[----------] \n",
      "Finished. Took 0.0631625652313 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495875759818\n",
      "New best!\n",
      "~Epoch 19~\n",
      "[----------] \n",
      "Finished. Took 0.0625838001569 minutes.\n",
      "Evaluating...\n",
      "Error = 0.556595433961\n",
      "~Epoch 20~\n",
      "[----------] \n",
      "Finished. Took 0.063525501887 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505302934351\n",
      "~Epoch 21~\n",
      "[----------] \n",
      "Finished. Took 0.063560116291 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507943025227\n",
      "~Epoch 22~\n",
      "[----------] \n",
      "Finished. Took 0.0634322484334 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515835735977\n",
      "~Epoch 23~\n",
      "[----------] \n",
      "Finished. Took 0.0624384522438 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49074268358\n",
      "New best!\n",
      "~Epoch 24~\n",
      "[----------] \n",
      "Finished. Took 0.0653397679329 minutes.\n",
      "Evaluating...\n",
      "Error = 0.490077644689\n",
      "New best!\n",
      "~Epoch 25~\n",
      "[----------] \n",
      "Finished. Took 0.0649476687113 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509862868181\n",
      "~Epoch 26~\n",
      "[----------] \n",
      "Finished. Took 0.0673483014107 minutes.\n",
      "Evaluating...\n",
      "Error = 0.491216913655\n",
      "~Epoch 27~\n",
      "[----------] \n",
      "Finished. Took 0.0618007659912 minutes.\n",
      "Evaluating...\n",
      "Error = 0.526977333708\n",
      "~Epoch 28~\n",
      "[----------] \n",
      "Finished. Took 0.0636481801669 minutes.\n",
      "Evaluating...\n",
      "Error = 0.491302998421\n",
      "~Epoch 29~\n",
      "[----------] \n",
      "Finished. Took 0.0629039843877 minutes.\n",
      "Evaluating...\n",
      "Error = 0.489571226124\n",
      "New best!\n",
      "~Epoch 30~\n",
      "[----------] \n",
      "Finished. Took 0.0673790693283 minutes.\n",
      "Evaluating...\n",
      "Error = 0.522661938001\n",
      "~Epoch 31~\n",
      "[----------] \n",
      "Finished. Took 0.0649219830831 minutes.\n",
      "Evaluating...\n",
      "Error = 0.484473411787\n",
      "New best!\n",
      "~Epoch 32~\n",
      "[----------] \n",
      "Finished. Took 0.0624222517014 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502920384586\n",
      "~Epoch 33~\n",
      "[----------] \n",
      "Finished. Took 0.0621073166529 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500842599912\n",
      "~Epoch 34~\n",
      "[----------] \n",
      "Finished. Took 0.0620632012685 minutes.\n",
      "Evaluating...\n",
      "Error = 0.48411159546\n",
      "New best!\n",
      "~Epoch 35~\n",
      "[----------] \n",
      "Finished. Took 0.061832066377 minutes.\n",
      "Evaluating...\n",
      "Error = 0.484809981814\n",
      "~Epoch 36~\n",
      "[----------] \n",
      "Finished. Took 0.06483831803 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498966319001\n",
      "~Epoch 37~\n",
      "[----------] \n",
      "Finished. Took 0.0651249170303 minutes.\n",
      "Evaluating...\n",
      "Error = 0.480578672594\n",
      "New best!\n",
      "~Epoch 38~\n",
      "[----------] \n",
      "Finished. Took 0.0637497504552 minutes.\n",
      "Evaluating...\n",
      "Error = 0.519266407322\n",
      "~Epoch 39~\n",
      "[----------] \n",
      "Finished. Took 0.0626449187597 minutes.\n",
      "Evaluating...\n",
      "Error = 0.510778413462\n",
      "~Epoch 40~\n",
      "[----------] \n",
      "Finished. Took 0.0627690672874 minutes.\n",
      "Evaluating...\n",
      "Error = 0.480981753801\n",
      "~Epoch 41~\n",
      "[----------] \n",
      "Finished. Took 0.0618949333827 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49735057953\n",
      "~Epoch 42~\n",
      "[----------] \n",
      "Finished. Took 0.0659536321958 minutes.\n",
      "Evaluating...\n",
      "Error = 0.494612901754\n",
      "~Epoch 43~\n",
      "[----------] \n",
      "Finished. Took 0.0650105675062 minutes.\n",
      "Evaluating...\n",
      "Error = 0.485002182366\n",
      "~Epoch 44~\n",
      "[----------] \n",
      "Finished. Took 0.0627195994059 minutes.\n",
      "Evaluating...\n",
      "Error = 0.516335595344\n",
      "~Epoch 45~\n",
      "[----------] \n",
      "Finished. Took 0.0631604512533 minutes.\n",
      "Evaluating...\n",
      "Error = 0.524498789046\n",
      "~Epoch 46~\n",
      "[----------] \n",
      "Finished. Took 0.0623507698377 minutes.\n",
      "Evaluating...\n",
      "Error = 0.492611140723\n",
      "~Epoch 47~\n",
      "[----------] \n",
      "Finished. Took 0.0627889513969 minutes.\n",
      "Evaluating...\n",
      "Error = 0.519609537001\n",
      "~Epoch 48~\n",
      "[----------] \n",
      "Finished. Took 0.0651953816414 minutes.\n",
      "Evaluating...\n",
      "Error = 0.525935220152\n",
      "~Epoch 49~\n",
      "[----------] \n",
      "Finished. Took 0.0633673350016 minutes.\n",
      "Evaluating...\n",
      "Error = 0.518712588866\n",
      "~Epoch 50~\n",
      "[----------] \n",
      "Finished. Took 0.0629025181135 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513994629451\n",
      "~Epoch 51~\n",
      "[----------] \n",
      "Finished. Took 0.0625971476237 minutes.\n",
      "Evaluating...\n",
      "Error = 0.511522706346\n",
      "~Epoch 52~\n",
      "[----------] \n",
      "Finished. Took 0.0621159831683 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501157819825\n",
      "~Epoch 53~\n",
      "[----------] \n",
      "Finished. Took 0.06209354798 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513246931567\n",
      "~Epoch 54~\n",
      "[----------] \n",
      "Finished. Took 0.0629115343094 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504309816997\n",
      "~Epoch 55~\n",
      "[----------] \n",
      "Finished. Took 0.0630170146624 minutes.\n",
      "Evaluating...\n",
      "Error = 0.520751572261\n",
      "~Epoch 56~\n",
      "[----------] \n",
      "Finished. Took 0.0637639005979 minutes.\n",
      "Evaluating...\n",
      "Error = 0.528334883778\n",
      "~Epoch 57~\n",
      "[----------] \n",
      "Finished. Took 0.0644819974899 minutes.\n",
      "Evaluating...\n",
      "Error = 0.529092862997\n",
      "~Epoch 58~\n",
      "[----------] \n",
      "Finished. Took 0.0624527851741 minutes.\n",
      "Evaluating...\n",
      "Error = 0.518062792342\n",
      "~Epoch 59~\n",
      "[----------] \n",
      "Finished. Took 0.0627436995506 minutes.\n",
      "Evaluating...\n",
      "Error = 0.52550188355\n",
      "~Epoch 60~\n",
      "[----------] \n",
      "Finished. Took 0.0619764844577 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515832693032\n",
      "~Epoch 61~\n",
      "[----------] \n",
      "Finished. Took 0.0629367510478 minutes.\n",
      "Evaluating...\n",
      "Error = 0.521677449666\n",
      "~Epoch 62~\n",
      "[----------] \n",
      "Finished. Took 0.0616936326027 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515866795873\n",
      "~Epoch 63~\n",
      "[----------] \n",
      "Finished. Took 0.061903933684 minutes.\n",
      "Evaluating...\n",
      "Error = 0.529823074656\n",
      "~Epoch 64~\n",
      "[----------] \n",
      "Finished. Took 0.0630147854487 minutes.\n",
      "Evaluating...\n",
      "Error = 0.542104697261\n",
      "~Epoch 65~\n",
      "[----------] \n",
      "Finished. Took 0.0621324499448 minutes.\n",
      "Evaluating...\n",
      "Error = 0.525578387248\n",
      "~Epoch 66~\n",
      "[----------] \n",
      "Finished. Took 0.062516884009 minutes.\n",
      "Evaluating...\n",
      "Error = 0.530900743806\n",
      "~Epoch 67~\n",
      "[----------] \n",
      "Finished. Took 0.0620643496513 minutes.\n",
      "Evaluating...\n",
      "Error = 0.522595416356\n",
      "~Epoch 68~\n",
      "[----------] \n",
      "Finished. Took 0.0622493823369 minutes.\n",
      "Evaluating...\n",
      "Error = 0.535861552772\n",
      "~Epoch 69~\n",
      "[----------] \n",
      "Finished. Took 0.0644143501918 minutes.\n",
      "Evaluating...\n",
      "Error = 0.541683685951\n",
      "~Epoch 70~\n",
      "[----------] \n",
      "Finished. Took 0.0628786007563 minutes.\n",
      "Evaluating...\n",
      "Error = 0.541278611883\n",
      "~Epoch 71~\n",
      "[----------] \n",
      "Finished. Took 0.0619149327278 minutes.\n",
      "Evaluating...\n",
      "Error = 0.535156529468\n",
      "~Epoch 72~\n",
      "[----------] \n",
      "Finished. Took 0.0618670145671 minutes.\n",
      "Evaluating...\n",
      "Error = 0.542247598663\n",
      "~Epoch 73~\n",
      "[----------] \n",
      "Finished. Took 0.0619095484416 minutes.\n",
      "Evaluating...\n",
      "Error = 0.551964634638\n",
      "~Epoch 74~\n",
      "[----------] \n",
      "Finished. Took 0.0624509970347 minutes.\n",
      "Evaluating...\n",
      "Error = 0.545390150888\n",
      "~Epoch 75~\n",
      "[----------] \n",
      "Finished. Took 0.0622654159864 minutes.\n",
      "Evaluating...\n",
      "Error = 0.530755063186\n",
      "~Epoch 76~\n",
      "[----------] \n",
      "Finished. Took 0.0623998006185 minutes.\n",
      "Evaluating...\n",
      "Error = 0.534114026421\n",
      "~Epoch 77~\n",
      "[----------] \n",
      "Finished. Took 0.0631959319115 minutes.\n",
      "Evaluating...\n",
      "Error = 0.529855331822\n",
      "Finished!\n",
      "Best error: 0.480578672594\n",
      "Checking...\n",
      "0 [-0.75661374] [-1]\n",
      "1 [-0.07125913] [1]\n",
      "2 [-0.10001692] [-1]\n",
      "3 [-0.34914696] [-1]\n",
      "4 [-0.70358137] [-1]\n",
      "5 [-0.46617552] [-1]\n",
      "6 [ 0.17452248] [1]\n",
      "Accuracy: train 0.737288135593 val 0.6 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# moar experiments; current baseline\n",
    "\n",
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model_b200 = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model_b200, s0[:200], s1[:200], s2[:200], y[:200], L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 596,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[----------] \n",
      "Finished. Took 0.0547526836395 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508826981799\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[----------] \n",
      "Finished. Took 0.0535140156746 minutes.\n",
      "Evaluating...\n",
      "Error = 0.563419099929\n",
      "~Epoch 3~\n",
      "[----------] \n",
      "Finished. Took 0.0528361479441 minutes.\n",
      "Evaluating...\n",
      "Error = 0.510709208079\n",
      "~Epoch 4~\n",
      "[----------] \n",
      "Finished. Took 0.0533079147339 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500050257785\n",
      "New best!\n",
      "~Epoch 5~\n",
      "[----------] \n",
      "Finished. Took 0.0530267477036 minutes.\n",
      "Evaluating...\n",
      "Error = 0.536795323265\n",
      "~Epoch 6~\n",
      "[----------] \n",
      "Finished. Took 0.053573067983 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517057083816\n",
      "~Epoch 7~\n",
      "[----------] \n",
      "Finished. Took 0.0535600821177 minutes.\n",
      "Evaluating...\n",
      "Error = 0.555145365423\n",
      "~Epoch 8~\n",
      "[----------] \n",
      "Finished. Took 0.0534473180771 minutes.\n",
      "Evaluating...\n",
      "Error = 0.521086990062\n",
      "~Epoch 9~\n",
      "[----------] \n",
      "Finished. Took 0.0530782183011 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50045284589\n",
      "~Epoch 10~\n",
      "[----------] \n",
      "Finished. Took 0.05291056633 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503590827112\n",
      "~Epoch 11~\n",
      "[----------] \n",
      "Finished. Took 0.0536769668261 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500570991308\n",
      "~Epoch 12~\n",
      "[----------] \n",
      "Finished. Took 0.0533007502556 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500095679951\n",
      "~Epoch 13~\n",
      "[----------] \n",
      "Finished. Took 0.053952050209 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502477172779\n",
      "~Epoch 14~\n",
      "[----------] \n",
      "Finished. Took 0.0534105817477 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509404922173\n",
      "~Epoch 15~\n",
      "[----------] \n",
      "Finished. Took 0.0534656325976 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499426324621\n",
      "New best!\n",
      "~Epoch 16~\n",
      "[----------] \n",
      "Finished. Took 0.0539198160172 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503287500335\n",
      "~Epoch 17~\n",
      "[----------] \n",
      "Finished. Took 0.0541399478912 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508341197643\n",
      "~Epoch 18~\n",
      "[----------] \n",
      "Finished. Took 0.0534942825635 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501582938283\n",
      "~Epoch 19~\n",
      "[----------] \n",
      "Finished. Took 0.0534730672836 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504869593764\n",
      "~Epoch 20~\n",
      "[----------] \n",
      "Finished. Took 0.0534022331238 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507201546907\n",
      "~Epoch 21~\n",
      "[----------] \n",
      "Finished. Took 0.0534822821617 minutes.\n",
      "Evaluating...\n",
      "Error = 0.574088183953\n",
      "~Epoch 22~\n",
      "[----------] \n",
      "Finished. Took 0.0531497001648 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505307415454\n",
      "~Epoch 23~\n",
      "[----------] \n",
      "Finished. Took 0.0532215674718 minutes.\n",
      "Evaluating...\n",
      "Error = 0.56934604164\n",
      "~Epoch 24~\n",
      "[----------] \n",
      "Finished. Took 0.0544077674548 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505855871725\n",
      "~Epoch 25~\n",
      "[----------] \n",
      "Finished. Took 0.0535349488258 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517942196873\n",
      "~Epoch 26~\n",
      "[----------] \n",
      "Finished. Took 0.0534862995148 minutes.\n",
      "Evaluating...\n",
      "Error = 0.532349470947\n",
      "~Epoch 27~\n",
      "[----------] \n",
      "Finished. Took 0.0535636862119 minutes.\n",
      "Evaluating...\n",
      "Error = 0.524064476093\n",
      "~Epoch 28~\n",
      "[----------] \n",
      "Finished. Took 0.0530575474103 minutes.\n",
      "Evaluating...\n",
      "Error = 0.524998500708\n",
      "~Epoch 29~\n",
      "[----------] \n",
      "Finished. Took 0.0544121305148 minutes.\n",
      "Evaluating...\n",
      "Error = 0.529380572652\n",
      "~Epoch 30~\n",
      "[----------] \n",
      "Finished. Took 0.0530993501345 minutes.\n",
      "Evaluating...\n",
      "Error = 0.536615427294\n",
      "~Epoch 31~\n",
      "[----------] \n",
      "Finished. Took 0.0541298985481 minutes.\n",
      "Evaluating...\n",
      "Error = 0.516985444089\n",
      "~Epoch 32~\n",
      "[----------] \n",
      "Finished. Took 0.0538256168365 minutes.\n",
      "Evaluating...\n",
      "Error = 0.521898643249\n",
      "~Epoch 33~\n",
      "[----------] \n",
      "Finished. Took 0.0530315677325 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515755953933\n",
      "~Epoch 34~\n",
      "[----------] \n",
      "Finished. Took 0.0532689332962 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517681574057\n",
      "~Epoch 35~\n",
      "[----------] \n",
      "Finished. Took 0.05428596735 minutes.\n",
      "Evaluating...\n",
      "Error = 0.546658278558\n",
      "~Epoch 36~\n",
      "[----------] \n",
      "Finished. Took 0.0535564144452 minutes.\n",
      "Evaluating...\n",
      "Error = 0.54927491292\n",
      "~Epoch 37~\n",
      "[----------] \n",
      "Finished. Took 0.0530047972997 minutes.\n",
      "Evaluating...\n",
      "Error = 0.524295957505\n",
      "~Epoch 38~\n",
      "[----------] \n",
      "Finished. Took 0.0532480518023 minutes.\n",
      "Evaluating...\n",
      "Error = 0.518056757397\n",
      "~Epoch 39~\n",
      "[----------] \n",
      "Finished. Took 0.0527019182841 minutes.\n",
      "Evaluating...\n",
      "Error = 0.533541662146\n",
      "~Epoch 40~\n",
      "[----------] \n",
      "Finished. Took 0.0529022653898 minutes.\n",
      "Evaluating...\n",
      "Error = 0.519301358738\n",
      "~Epoch 41~\n",
      "[----------] \n",
      "Finished. Took 0.0528719345729 minutes.\n",
      "Evaluating...\n",
      "Error = 0.516300890536\n",
      "~Epoch 42~\n",
      "[----------] \n",
      "Finished. Took 0.0546480178833 minutes.\n",
      "Evaluating...\n",
      "Error = 0.521021907803\n",
      "~Epoch 43~\n",
      "[----------] \n",
      "Finished. Took 0.0529564857483 minutes.\n",
      "Evaluating...\n",
      "Error = 0.521071237988\n",
      "~Epoch 44~\n",
      "[----------] \n",
      "Finished. Took 0.0528843641281 minutes.\n",
      "Evaluating...\n",
      "Error = 0.524452496487\n",
      "~Epoch 45~\n",
      "[----------] \n",
      "Finished. Took 0.0536082983017 minutes.\n",
      "Evaluating...\n",
      "Error = 0.525182480205\n",
      "~Epoch 46~\n",
      "[----------] \n",
      "Finished. Took 0.0528624494871 minutes.\n",
      "Evaluating...\n",
      "Error = 0.543302809996\n",
      "~Epoch 47~\n",
      "[----------] \n",
      "Finished. Took 0.0537150979042 minutes.\n",
      "Evaluating...\n",
      "Error = 0.556469357076\n",
      "~Epoch 48~\n",
      "[----------] \n",
      "Finished. Took 0.0529933691025 minutes.\n",
      "Evaluating...\n",
      "Error = 0.542394345628\n",
      "~Epoch 49~\n",
      "[----------] \n",
      "Finished. Took 0.0531909863154 minutes.\n",
      "Evaluating...\n",
      "Error = 0.536082284496\n",
      "~Epoch 50~\n",
      "[----------] \n",
      "Finished. Took 0.0532248139381 minutes.\n",
      "Evaluating...\n",
      "Error = 0.570006752373\n",
      "~Epoch 51~\n",
      "[----------] \n",
      "Finished. Took 0.0531095663706 minutes.\n",
      "Evaluating...\n",
      "Error = 0.538640052758\n",
      "~Epoch 52~\n",
      "[----------] \n",
      "Finished. Took 0.0530679146449 minutes.\n",
      "Evaluating...\n",
      "Error = 0.555000725816\n",
      "~Epoch 53~\n",
      "[----------] \n",
      "Finished. Took 0.0528869350751 minutes.\n",
      "Evaluating...\n",
      "Error = 0.553994348058\n",
      "~Epoch 54~\n",
      "[----------] \n",
      "Finished. Took 0.0536607146263 minutes.\n",
      "Evaluating...\n",
      "Error = 0.555862094676\n",
      "~Epoch 55~\n",
      "[----------] \n",
      "Finished. Took 0.05344196558 minutes.\n",
      "Evaluating...\n",
      "Error = 0.546088068872\n",
      "Finished!\n",
      "Best error: 0.499426324621\n",
      "Checking...\n",
      "0 [-0.22941303] [-1]\n",
      "1 [-0.02929596] [1]\n",
      "2 [-0.10172462] [-1]\n",
      "3 [-0.05090393] [-1]\n",
      "4 [-0.13614282] [-1]\n",
      "5 [-0.08967815] [-1]\n",
      "6 [ 0.03839583] [1]\n",
      "Accuracy: train 0.648305084746 val 0.433333333333 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# moar experiments\n",
    "\n",
    "# XXX: this is invalid way to specify activation_func, why did it have an effect?\n",
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, activation_func=nnb.activation.tanh)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, activation_func=nnb.activation.tanh)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, activation_func=nnb.activation.tanh)[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model, s0[:200], s1[:200], s2[:200], y[:200], L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 597,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[----------] \n",
      "Finished. Took 0.058017885685 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506859348519\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[----------] \n",
      "Finished. Took 0.0582812468211 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513755296363\n",
      "~Epoch 3~\n",
      "[----------] \n",
      "Finished. Took 0.0579884489377 minutes.\n",
      "Evaluating...\n",
      "Error = 0.519423497121\n",
      "~Epoch 4~\n",
      "[----------] \n",
      "Finished. Took 0.0583023667336 minutes.\n",
      "Evaluating...\n",
      "Error = 0.52423108911\n",
      "~Epoch 5~\n",
      "[----------] \n",
      "Finished. Took 0.0586972633998 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502146940568\n",
      "New best!\n",
      "~Epoch 6~\n",
      "[----------] \n",
      "Finished. Took 0.0581946333249 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501618242121\n",
      "New best!\n",
      "~Epoch 7~\n",
      "[----------] \n",
      "Finished. Took 0.058636935552 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50427002724\n",
      "~Epoch 8~\n",
      "[----------] \n",
      "Finished. Took 0.0585767865181 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500344790499\n",
      "New best!\n",
      "~Epoch 9~\n",
      "[----------] \n",
      "Finished. Took 0.0595157027245 minutes.\n",
      "Evaluating...\n",
      "Error = 0.541830910127\n",
      "~Epoch 10~\n",
      "[----------] \n",
      "Finished. Took 0.0588829517365 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501934183501\n",
      "~Epoch 11~\n",
      "[----------] \n",
      "Finished. Took 0.0583931803703 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501247402149\n",
      "~Epoch 12~\n",
      "[----------] \n",
      "Finished. Took 0.0581096847852 minutes.\n",
      "Evaluating...\n",
      "Error = 0.52297783783\n",
      "~Epoch 13~\n",
      "[----------] \n",
      "Finished. Took 0.0586202859879 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504382023025\n",
      "~Epoch 14~\n",
      "[----------] \n",
      "Finished. Took 0.0589628815651 minutes.\n",
      "Evaluating...\n",
      "Error = 0.51977880853\n",
      "~Epoch 15~\n",
      "[----------] \n",
      "Finished. Took 0.0587866306305 minutes.\n",
      "Evaluating...\n",
      "Error = 0.512062805874\n",
      "~Epoch 16~\n",
      "[----------] \n",
      "Finished. Took 0.0579735636711 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508320498528\n",
      "~Epoch 17~\n",
      "[----------] \n",
      "Finished. Took 0.0581330339114 minutes.\n",
      "Evaluating...\n",
      "Error = 0.542077638786\n",
      "~Epoch 18~\n",
      "[----------] \n",
      "Finished. Took 0.0581241488457 minutes.\n",
      "Evaluating...\n",
      "Error = 0.512380793419\n",
      "~Epoch 19~\n",
      "[----------] \n",
      "Finished. Took 0.0581393837929 minutes.\n",
      "Evaluating...\n",
      "Error = 0.516475841795\n",
      "~Epoch 20~\n",
      "[----------] \n",
      "Finished. Took 0.0584867676099 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517318900217\n",
      "~Epoch 21~\n",
      "[----------] \n",
      "Finished. Took 0.059091881911 minutes.\n",
      "Evaluating...\n",
      "Error = 0.55189599742\n",
      "~Epoch 22~\n",
      "[----------] \n",
      "Finished. Took 0.0586347341537 minutes.\n",
      "Evaluating...\n",
      "Error = 0.527077906622\n",
      "~Epoch 23~\n",
      "[----------] \n",
      "Finished. Took 0.0579918861389 minutes.\n",
      "Evaluating...\n",
      "Error = 0.522572799546\n",
      "~Epoch 24~\n",
      "[----------] \n",
      "Finished. Took 0.0583678523699 minutes.\n",
      "Evaluating...\n",
      "Error = 0.543960273498\n",
      "~Epoch 25~\n",
      "[----------] \n",
      "Finished. Took 0.059153898557 minutes.\n",
      "Evaluating...\n",
      "Error = 0.554646487159\n",
      "~Epoch 26~\n",
      "[----------] \n",
      "Finished. Took 0.0590711673101 minutes.\n",
      "Evaluating...\n",
      "Error = 0.555367029713\n",
      "~Epoch 27~\n",
      "[----------] \n",
      "Finished. Took 0.0584856152534 minutes.\n",
      "Evaluating...\n",
      "Error = 0.531005344372\n",
      "~Epoch 28~\n",
      "[----------] \n",
      "Finished. Took 0.0583397189776 minutes.\n",
      "Evaluating...\n",
      "Error = 0.52591789387\n",
      "~Epoch 29~\n",
      "[----------] \n",
      "Finished. Took 0.0582184354464 minutes.\n",
      "Evaluating...\n",
      "Error = 0.559058950006\n",
      "~Epoch 30~\n",
      "[----------] \n",
      "Finished. Took 0.0583799521128 minutes.\n",
      "Evaluating...\n",
      "Error = 0.538122515969\n",
      "~Epoch 31~\n",
      "[----------] \n",
      "Finished. Took 0.0591865499814 minutes.\n",
      "Evaluating...\n",
      "Error = 0.512895715328\n",
      "~Epoch 32~\n",
      "[----------] \n",
      "Finished. Took 0.0583586851756 minutes.\n",
      "Evaluating...\n",
      "Error = 0.550046513702\n",
      "~Epoch 33~\n",
      "[----------] \n",
      "Finished. Took 0.0583722988764 minutes.\n",
      "Evaluating...\n",
      "Error = 0.52884707576\n",
      "~Epoch 34~\n",
      "[----------] \n",
      "Finished. Took 0.0591872652372 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508805081123\n",
      "~Epoch 35~\n",
      "[----------] \n",
      "Finished. Took 0.058367617925 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506979327877\n",
      "~Epoch 36~\n",
      "[----------] \n",
      "Finished. Took 0.0583352128665 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503862729849\n",
      "~Epoch 37~\n",
      "[----------] \n",
      "Finished. Took 0.0583172520002 minutes.\n",
      "Evaluating...\n",
      "Error = 0.563252022187\n",
      "~Epoch 38~\n",
      "[----------] \n",
      "Finished. Took 0.0582687695821 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502411257901\n",
      "~Epoch 39~\n",
      "[----------] \n",
      "Finished. Took 0.0582683841387 minutes.\n",
      "Evaluating...\n",
      "Error = 0.487685510369\n",
      "New best!\n",
      "~Epoch 40~\n",
      "[----------] \n",
      "Finished. Took 0.0581397970517 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496600671523\n",
      "~Epoch 41~\n",
      "[----------] \n",
      "Finished. Took 0.0590082645416 minutes.\n",
      "Evaluating...\n",
      "Error = 0.488113053018\n",
      "~Epoch 42~\n",
      "[----------] \n",
      "Finished. Took 0.0584957838058 minutes.\n",
      "Evaluating...\n",
      "Error = 0.514561115361\n",
      "~Epoch 43~\n",
      "[----------] \n",
      "Finished. Took 0.0584136644999 minutes.\n",
      "Evaluating...\n",
      "Error = 0.555738731712\n",
      "~Epoch 44~\n",
      "[----------] \n",
      "Finished. Took 0.0586799184481 minutes.\n",
      "Evaluating...\n",
      "Error = 0.48031429746\n",
      "New best!\n",
      "~Epoch 45~\n",
      "[----------] \n",
      "Finished. Took 0.0591688990593 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495098317508\n",
      "~Epoch 46~\n",
      "[----------] \n",
      "Finished. Took 0.0589003682137 minutes.\n",
      "Evaluating...\n",
      "Error = 0.476122475349\n",
      "New best!\n",
      "~Epoch 47~\n",
      "[----------] \n",
      "Finished. Took 0.0597505648931 minutes.\n",
      "Evaluating...\n",
      "Error = 0.486640349667\n",
      "~Epoch 48~\n",
      "[----------] \n",
      "Finished. Took 0.0585248629252 minutes.\n",
      "Evaluating...\n",
      "Error = 0.470267069972\n",
      "New best!\n",
      "~Epoch 49~\n",
      "[----------] \n",
      "Finished. Took 0.058840016524 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517262531169\n",
      "~Epoch 50~\n",
      "[----------] \n",
      "Finished. Took 0.0589732329051 minutes.\n",
      "Evaluating...\n",
      "Error = 0.492153285683\n",
      "~Epoch 51~\n",
      "[----------] \n",
      "Finished. Took 0.0589861512184 minutes.\n",
      "Evaluating...\n",
      "Error = 0.52194517741\n",
      "~Epoch 52~\n",
      "[----------] \n",
      "Finished. Took 0.0591838359833 minutes.\n",
      "Evaluating...\n",
      "Error = 0.491064529002\n",
      "~Epoch 53~\n",
      "[----------] \n",
      "Finished. Took 0.0584985812505 minutes.\n",
      "Evaluating...\n",
      "Error = 0.466644956589\n",
      "New best!\n",
      "~Epoch 54~\n",
      "[----------] \n",
      "Finished. Took 0.0589245676994 minutes.\n",
      "Evaluating...\n",
      "Error = 0.467892077812\n",
      "~Epoch 55~\n",
      "[----------] \n",
      "Finished. Took 0.0582352677981 minutes.\n",
      "Evaluating...\n",
      "Error = 0.472275079961\n",
      "~Epoch 56~\n",
      "[----------] \n",
      "Finished. Took 0.0587860186895 minutes.\n",
      "Evaluating...\n",
      "Error = 0.471059253938\n",
      "~Epoch 57~\n",
      "[----------] \n",
      "Finished. Took 0.0587165673574 minutes.\n",
      "Evaluating...\n",
      "Error = 0.48476900328\n",
      "~Epoch 58~\n",
      "[----------] \n",
      "Finished. Took 0.0581586321195 minutes.\n",
      "Evaluating...\n",
      "Error = 0.476080470825\n",
      "~Epoch 59~\n",
      "[----------] \n",
      "Finished. Took 0.0587845842044 minutes.\n",
      "Evaluating...\n",
      "Error = 0.481412518337\n",
      "~Epoch 60~\n",
      "[----------] \n",
      "Finished. Took 0.059200934569 minutes.\n",
      "Evaluating...\n",
      "Error = 0.479866999185\n",
      "~Epoch 61~\n",
      "[----------] \n",
      "Finished. Took 0.058532734712 minutes.\n",
      "Evaluating...\n",
      "Error = 0.479813019538\n",
      "~Epoch 62~\n",
      "[----------] \n",
      "Finished. Took 0.0589085658391 minutes.\n",
      "Evaluating...\n",
      "Error = 0.482863726958\n",
      "~Epoch 63~\n",
      "[----------] \n",
      "Finished. Took 0.0595225016276 minutes.\n",
      "Evaluating...\n",
      "Error = 0.490348210721\n",
      "~Epoch 64~\n",
      "[----------] \n",
      "Finished. Took 0.0580527981122 minutes.\n",
      "Evaluating...\n",
      "Error = 0.486708687657\n",
      "~Epoch 65~\n",
      "[----------] \n",
      "Finished. Took 0.058454867204 minutes.\n",
      "Evaluating...\n",
      "Error = 0.490301417142\n",
      "~Epoch 66~\n",
      "[----------] \n",
      "Finished. Took 0.0592662016551 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499992007852\n",
      "~Epoch 67~\n",
      "[----------] \n",
      "Finished. Took 0.0583718816439 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505186624171\n",
      "~Epoch 68~\n",
      "[----------] \n",
      "Finished. Took 0.0585921009382 minutes.\n",
      "Evaluating...\n",
      "Error = 0.493104651362\n",
      "~Epoch 69~\n",
      "[----------] \n",
      "Finished. Took 0.0585879484812 minutes.\n",
      "Evaluating...\n",
      "Error = 0.519612344107\n",
      "~Epoch 70~\n",
      "[----------] \n",
      "Finished. Took 0.0584575653076 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503376150753\n",
      "~Epoch 71~\n",
      "[----------] \n",
      "Finished. Took 0.0588568329811 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513142025305\n",
      "~Epoch 72~\n",
      "[----------] \n",
      "Finished. Took 0.0592919985453 minutes.\n",
      "Evaluating...\n",
      "Error = 0.521872835328\n",
      "~Epoch 73~\n",
      "[----------] \n",
      "Finished. Took 0.0590180158615 minutes.\n",
      "Evaluating...\n",
      "Error = 0.516189658947\n",
      "~Epoch 74~\n",
      "[----------] \n",
      "Finished. Took 0.0595034321149 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517895389046\n",
      "~Epoch 75~\n",
      "[----------] \n",
      "Finished. Took 0.0590086817741 minutes.\n",
      "Evaluating...\n",
      "Error = 0.524693243326\n",
      "~Epoch 76~\n",
      "[----------] \n",
      "Finished. Took 0.0582903663317 minutes.\n",
      "Evaluating...\n",
      "Error = 0.522488823168\n",
      "~Epoch 77~\n",
      "[----------] \n",
      "Finished. Took 0.0585213502248 minutes.\n",
      "Evaluating...\n",
      "Error = 0.528681698943\n",
      "~Epoch 78~\n",
      "[----------] \n",
      "Finished. Took 0.0581918001175 minutes.\n",
      "Evaluating...\n",
      "Error = 0.530030010787\n",
      "~Epoch 79~\n",
      "[----------] \n",
      "Finished. Took 0.0594781001409 minutes.\n",
      "Evaluating...\n",
      "Error = 0.541289575726\n",
      "~Epoch 80~\n",
      "[----------] \n",
      "Finished. Took 0.0579391002655 minutes.\n",
      "Evaluating...\n",
      "Error = 0.537098006638\n",
      "~Epoch 81~\n",
      "[----------] \n",
      "Finished. Took 0.0583710511525 minutes.\n",
      "Evaluating...\n",
      "Error = 0.542340200054\n",
      "~Epoch 82~\n",
      "[----------] \n",
      "Finished. Took 0.0585532188416 minutes.\n",
      "Evaluating...\n",
      "Error = 0.553653425204\n",
      "~Epoch 83~\n",
      "[----------] \n",
      "Finished. Took 0.0575254162153 minutes.\n",
      "Evaluating...\n",
      "Error = 0.550702382207\n",
      "~Epoch 84~\n",
      "[----------] \n",
      "Finished. Took 0.0586851994197 minutes.\n",
      "Evaluating...\n",
      "Error = 0.55217211785\n",
      "~Epoch 85~\n",
      "[----------] \n",
      "Finished. Took 0.0586746692657 minutes.\n",
      "Evaluating...\n",
      "Error = 0.559603086097\n",
      "~Epoch 86~\n",
      "[----------] \n",
      "Finished. Took 0.0582661310832 minutes.\n",
      "Evaluating...\n",
      "Error = 0.553852956828\n",
      "~Epoch 87~\n",
      "[----------] \n",
      "Finished. Took 0.0586360017459 minutes.\n",
      "Evaluating...\n",
      "Error = 0.558160930332\n",
      "~Epoch 88~\n",
      "[----------] \n",
      "Finished. Took 0.0585920174917 minutes.\n",
      "Evaluating...\n",
      "Error = 0.568999444694\n",
      "~Epoch 89~\n",
      "[----------] \n",
      "Finished. Took 0.0583118319511 minutes.\n",
      "Evaluating...\n",
      "Error = 0.561259081902\n",
      "~Epoch 90~\n",
      "[----------] \n",
      "Finished. Took 0.0579925815264 minutes.\n",
      "Evaluating...\n",
      "Error = 0.563578845496\n",
      "~Epoch 91~\n",
      "[----------] \n",
      "Finished. Took 0.058042649428 minutes.\n",
      "Evaluating...\n",
      "Error = 0.577613937948\n",
      "~Epoch 92~\n",
      "[----------] \n",
      "Finished. Took 0.0582056164742 minutes.\n",
      "Evaluating...\n",
      "Error = 0.575879889232\n",
      "~Epoch 93~\n",
      "[----------] \n",
      "Finished. Took 0.0584018190702 minutes.\n",
      "Evaluating...\n",
      "Error = 0.591984461244\n",
      "Finished!\n",
      "Best error: 0.466644956589\n",
      "Checking...\n",
      "0 [-0.95967746] [-1]\n",
      "1 [-0.34099181] [1]\n",
      "2 [-0.04702698] [-1]\n",
      "3 [-0.69670311] [-1]\n",
      "4 [-0.97057908] [-1]\n",
      "5 [-0.84456242] [-1]\n",
      "6 [-0.08035553] [1]\n",
      "Accuracy: train 0.71186440678 val 0.633333333333 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# moar experiments\n",
    "\n",
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model, s0[:200], s1[:200], s2[:200], y[:200], L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 598,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[----------] \n",
      "Finished. Took 0.0552168647448 minutes.\n",
      "Evaluating...\n",
      "Error = 0.533162661501\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[----------] \n",
      "Finished. Took 0.0548652648926 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500166086692\n",
      "New best!\n",
      "~Epoch 3~\n",
      "[----------] \n",
      "Finished. Took 0.0550167520841 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500813871845\n",
      "~Epoch 4~\n",
      "[----------] \n",
      "Finished. Took 0.0550339698792 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499317301639\n",
      "New best!\n",
      "~Epoch 5~\n",
      "[----------] \n",
      "Finished. Took 0.0544922828674 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499207956887\n",
      "New best!\n",
      "~Epoch 6~\n",
      "[----------] \n",
      "Finished. Took 0.056290547053 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49893973001\n",
      "New best!\n",
      "~Epoch 7~\n",
      "[----------] \n",
      "Finished. Took 0.0557192325592 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505432846863\n",
      "~Epoch 8~\n",
      "[----------] \n",
      "Finished. Took 0.0547106822332 minutes.\n",
      "Evaluating...\n",
      "Error = 0.529741650527\n",
      "~Epoch 9~\n",
      "[----------] \n",
      "Finished. Took 0.0554574847221 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501003785981\n",
      "~Epoch 10~\n",
      "[----------] \n",
      "Finished. Took 0.0551352659861 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505765215004\n",
      "~Epoch 11~\n",
      "[----------] \n",
      "Finished. Took 0.0553090333939 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497400799839\n",
      "New best!\n",
      "~Epoch 12~\n",
      "[----------] \n",
      "Finished. Took 0.0549273173014 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499726845834\n",
      "~Epoch 13~\n",
      "[----------] \n",
      "Finished. Took 0.0549070159594 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495514315321\n",
      "New best!\n",
      "~Epoch 14~\n",
      "[----------] \n",
      "Finished. Took 0.0548584858576 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49713744847\n",
      "~Epoch 15~\n",
      "[----------] \n",
      "Finished. Took 0.0555760820707 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496318690469\n",
      "~Epoch 16~\n",
      "[----------] \n",
      "Finished. Took 0.0554111838341 minutes.\n",
      "Evaluating...\n",
      "Error = 0.534451750371\n",
      "~Epoch 17~\n",
      "[----------] \n",
      "Finished. Took 0.0549828847249 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500243226333\n",
      "~Epoch 18~\n",
      "[----------] \n",
      "Finished. Took 0.0548144658407 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495547497031\n",
      "~Epoch 19~\n",
      "[----------] \n",
      "Finished. Took 0.0549335837364 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498571736236\n",
      "~Epoch 20~\n",
      "[----------] \n",
      "Finished. Took 0.0549420992533 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497710321485\n",
      "~Epoch 21~\n",
      "[----------] \n",
      "Finished. Took 0.0554988821348 minutes.\n",
      "Evaluating...\n",
      "Error = 0.535441021184\n",
      "~Epoch 22~\n",
      "[----------] \n",
      "Finished. Took 0.0549632469813 minutes.\n",
      "Evaluating...\n",
      "Error = 0.516792162708\n",
      "~Epoch 23~\n",
      "[----------] \n",
      "Finished. Took 0.0551264643669 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499890645066\n",
      "~Epoch 24~\n",
      "[----------] \n",
      "Finished. Took 0.0549514492353 minutes.\n",
      "Evaluating...\n",
      "Error = 0.523984578939\n",
      "~Epoch 25~\n",
      "[----------] \n",
      "Finished. Took 0.0554434657097 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504586418644\n",
      "~Epoch 26~\n",
      "[----------] \n",
      "Finished. Took 0.0549416502317 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502915912035\n",
      "~Epoch 27~\n",
      "[----------] \n",
      "Finished. Took 0.0552639524142 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515381722799\n",
      "~Epoch 28~\n",
      "[----------] \n",
      "Finished. Took 0.0547629316648 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505460363203\n",
      "~Epoch 29~\n",
      "[----------] \n",
      "Finished. Took 0.055023765564 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501579186968\n",
      "~Epoch 30~\n",
      "[----------] \n",
      "Finished. Took 0.0551281809807 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499515145433\n",
      "~Epoch 31~\n",
      "[----------] \n",
      "Finished. Took 0.0549847642581 minutes.\n",
      "Evaluating...\n",
      "Error = 0.511739370257\n",
      "~Epoch 32~\n",
      "[----------] \n",
      "Finished. Took 0.0554240822792 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502945435279\n",
      "~Epoch 33~\n",
      "[----------] \n",
      "Finished. Took 0.0550068497658 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505794998408\n",
      "~Epoch 34~\n",
      "[----------] \n",
      "Finished. Took 0.0550788839658 minutes.\n",
      "Evaluating...\n",
      "Error = 0.561712384935\n",
      "~Epoch 35~\n",
      "[----------] \n",
      "Finished. Took 0.0549510161082 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505884573837\n",
      "~Epoch 36~\n",
      "[----------] \n",
      "Finished. Took 0.0552644491196 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50820420965\n",
      "~Epoch 37~\n",
      "[----------] \n",
      "Finished. Took 0.0552496194839 minutes.\n",
      "Evaluating...\n",
      "Error = 0.525174492656\n",
      "~Epoch 38~\n",
      "[----------] \n",
      "Finished. Took 0.0549059669177 minutes.\n",
      "Evaluating...\n",
      "Error = 0.555612209952\n",
      "~Epoch 39~\n",
      "[----------] \n",
      "Finished. Took 0.0552509466807 minutes.\n",
      "Evaluating...\n",
      "Error = 0.514754814492\n",
      "~Epoch 40~\n",
      "[----------] \n",
      "Finished. Took 0.0547700842222 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509951609624\n",
      "~Epoch 41~\n",
      "[----------] \n",
      "Finished. Took 0.0548504511515 minutes.\n",
      "Evaluating...\n",
      "Error = 0.54781660551\n",
      "~Epoch 42~\n",
      "[----------] \n",
      "Finished. Took 0.0549748341242 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513684184319\n",
      "~Epoch 43~\n",
      "[----------] \n",
      "Finished. Took 0.0549840648969 minutes.\n",
      "Evaluating...\n",
      "Error = 0.530621655191\n",
      "~Epoch 44~\n",
      "[----------] \n",
      "Finished. Took 0.0549992839495 minutes.\n",
      "Evaluating...\n",
      "Error = 0.523109102064\n",
      "~Epoch 45~\n",
      "[----------] \n",
      "Finished. Took 0.0547053813934 minutes.\n",
      "Evaluating...\n",
      "Error = 0.548698661902\n",
      "~Epoch 46~\n",
      "[----------] \n",
      "Finished. Took 0.0549272020658 minutes.\n",
      "Evaluating...\n",
      "Error = 0.558586147756\n",
      "~Epoch 47~\n",
      "[----------] \n",
      "Finished. Took 0.0548881491025 minutes.\n",
      "Evaluating...\n",
      "Error = 0.532063339511\n",
      "~Epoch 48~\n",
      "[----------] \n",
      "Finished. Took 0.0555996696154 minutes.\n",
      "Evaluating...\n",
      "Error = 0.53441085239\n",
      "~Epoch 49~\n",
      "[----------] \n",
      "Finished. Took 0.0548027674357 minutes.\n",
      "Evaluating...\n",
      "Error = 0.535187969923\n",
      "~Epoch 50~\n",
      "[----------] \n",
      "Finished. Took 0.0548443992933 minutes.\n",
      "Evaluating...\n",
      "Error = 0.5345930865\n",
      "~Epoch 51~\n",
      "[----------] \n",
      "Finished. Took 0.0546113173167 minutes.\n",
      "Evaluating...\n",
      "Error = 0.545855587249\n",
      "~Epoch 52~\n",
      "[----------] \n",
      "Finished. Took 0.0550095160802 minutes.\n",
      "Evaluating...\n",
      "Error = 0.538783080609\n",
      "~Epoch 53~\n",
      "[----------] \n",
      "Finished. Took 0.0553351640701 minutes.\n",
      "Evaluating...\n",
      "Error = 0.543459270312\n",
      "Finished!\n",
      "Best error: 0.495514315321\n",
      "Checking...\n",
      "0 [-0.13456601] [-1]\n",
      "1 [ 0.06374726] [1]\n",
      "2 [ 0.00468363] [-1]\n",
      "3 [-0.02367741] [-1]\n",
      "4 [-0.07976494] [-1]\n",
      "5 [-0.09925239] [-1]\n",
      "6 [ 0.01011162] [1]\n",
      "Accuracy: train 0.686440677966 val 0.566666666667 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# moar experiments\n",
    "\n",
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=5)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=5)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=5)[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*5, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model, s0[:200], s1[:200], s2[:200], y[:200], L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[----------] \n",
      "Finished. Took 0.0639068007469 minutes.\n",
      "Evaluating...\n",
      "Error = 0.52367817288\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[----------] \n",
      "Finished. Took 0.0626420338949 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509335139541\n",
      "New best!\n",
      "~Epoch 3~\n",
      "[----------] \n",
      "Finished. Took 0.0624714811643 minutes.\n",
      "Evaluating...\n",
      "Error = 0.552403016943\n",
      "~Epoch 4~\n",
      "[----------] \n",
      "Finished. Took 0.0626221179962 minutes.\n",
      "Evaluating...\n",
      "Error = 0.517385515971\n",
      "~Epoch 5~\n",
      "[----------] \n",
      "Finished. Took 0.0631368517876 minutes.\n",
      "Evaluating...\n",
      "Error = 0.538668932572\n",
      "~Epoch 6~\n",
      "[----------] \n",
      "Finished. Took 0.063644250234 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500204302745\n",
      "New best!\n",
      "~Epoch 7~\n",
      "[----------] \n",
      "Finished. Took 0.0627485990524 minutes.\n",
      "Evaluating...\n",
      "Error = 0.511323317212\n",
      "~Epoch 8~\n",
      "[----------] \n",
      "Finished. Took 0.0628132184347 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503261687349\n",
      "~Epoch 9~\n",
      "[----------] \n",
      "Finished. Took 0.0629855155945 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502552247495\n",
      "~Epoch 10~\n",
      "[----------] \n",
      "Finished. Took 0.0638203660647 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502357130848\n",
      "~Epoch 11~\n",
      "[----------] \n",
      "Finished. Took 0.0634811162949 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502603560617\n",
      "~Epoch 12~\n",
      "[----------] \n",
      "Finished. Took 0.0635478019714 minutes.\n",
      "Evaluating...\n",
      "Error = 0.516972693609\n",
      "~Epoch 13~\n",
      "[----------] \n",
      "Finished. Took 0.0635185162226 minutes.\n",
      "Evaluating...\n",
      "Error = 0.511532718527\n",
      "~Epoch 14~\n",
      "[----------] \n",
      "Finished. Took 0.0631422321002 minutes.\n",
      "Evaluating...\n",
      "Error = 0.511547987336\n",
      "~Epoch 15~\n",
      "[----------] \n",
      "Finished. Took 0.0635190844536 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513717783101\n",
      "~Epoch 16~\n",
      "[----------] \n",
      "Finished. Took 0.0624424854914 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515902623185\n",
      "~Epoch 17~\n",
      "[----------] \n",
      "Finished. Took 0.0641491015752 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501924825642\n",
      "~Epoch 18~\n",
      "[----------] \n",
      "Finished. Took 0.0635773181915 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502487024402\n",
      "~Epoch 19~\n",
      "[----------] \n",
      "Finished. Took 0.0640576640765 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501845643386\n",
      "~Epoch 20~\n",
      "[----------] \n",
      "Finished. Took 0.0628087679545 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504604963433\n",
      "~Epoch 21~\n",
      "[----------] \n",
      "Finished. Took 0.0635331988335 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501932037391\n",
      "~Epoch 22~\n",
      "[----------] \n",
      "Finished. Took 0.0644774357478 minutes.\n",
      "Evaluating...\n",
      "Error = 0.522718391868\n",
      "~Epoch 23~\n",
      "[----------] \n",
      "Finished. Took 0.0633525172869 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501534523844\n",
      "~Epoch 24~\n",
      "[----------] \n",
      "Finished. Took 0.0634623169899 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501667955731\n",
      "~Epoch 25~\n",
      "[----------] \n",
      "Finished. Took 0.0634546677272 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502036222845\n",
      "~Epoch 26~\n",
      "[----------] \n",
      "Finished. Took 0.0640072345734 minutes.\n",
      "Evaluating...\n",
      "Error = 0.512660383966\n",
      "~Epoch 27~\n",
      "[----------] \n",
      "Finished. Took 0.0640907804171 minutes.\n",
      "Evaluating...\n",
      "Error = 0.510309541296\n",
      "~Epoch 28~\n",
      "[----------] \n",
      "Finished. Took 0.0646317164103 minutes.\n",
      "Evaluating...\n",
      "Error = 0.56343186372\n",
      "~Epoch 29~\n",
      "[----------] \n",
      "Finished. Took 0.0645444194476 minutes.\n",
      "Evaluating...\n",
      "Error = 0.52168420547\n",
      "~Epoch 30~\n",
      "[----------] \n",
      "Finished. Took 0.0631102363269 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502974306183\n",
      "~Epoch 31~\n",
      "[----------] \n",
      "Finished. Took 0.0634703834852 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503280116493\n",
      "~Epoch 32~\n",
      "[----------] \n",
      "Finished. Took 0.0636902848879 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504020113102\n",
      "~Epoch 33~\n",
      "[----------] \n",
      "Finished. Took 0.0648260633151 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505278653514\n",
      "~Epoch 34~\n",
      "[----------] \n",
      "Finished. Took 0.0638102849325 minutes.\n",
      "Evaluating...\n",
      "Error = 0.519021740274\n",
      "~Epoch 35~\n",
      "[----------] \n",
      "Finished. Took 0.0636963804563 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506237107635\n",
      "~Epoch 36~\n",
      "[----------] \n",
      "Finished. Took 0.0631655971209 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507836384028\n",
      "~Epoch 37~\n",
      "[----------] \n",
      "Finished. Took 0.064791850249 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505402161201\n",
      "~Epoch 38~\n",
      "[----------] \n",
      "Finished. Took 0.0642714818319 minutes.\n",
      "Evaluating...\n",
      "Error = 0.512700346073\n",
      "~Epoch 39~\n",
      "[----------] \n",
      "Finished. Took 0.0639438350995 minutes.\n",
      "Evaluating...\n",
      "Error = 0.543515367246\n",
      "~Epoch 40~\n",
      "[----------] \n",
      "Finished. Took 0.0633761008581 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507084394495\n",
      "~Epoch 41~\n",
      "[----------] \n",
      "Finished. Took 0.0627504150073 minutes.\n",
      "Evaluating...\n",
      "Error = 0.514161997476\n",
      "~Epoch 42~\n",
      "[----------] \n",
      "Finished. Took 0.063713033994 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507520828835\n",
      "~Epoch 43~\n",
      "[----------] \n",
      "Finished. Took 0.0633141318957 minutes.\n",
      "Evaluating...\n",
      "Error = 0.527130515006\n",
      "~Epoch 44~\n",
      "[----------] \n",
      "Finished. Took 0.0631861845652 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515331078767\n",
      "~Epoch 45~\n",
      "[----------] \n",
      "Finished. Took 0.062879216671 minutes.\n",
      "Evaluating...\n",
      "Error = 0.520688334998\n",
      "~Epoch 46~\n",
      "[----------] \n",
      "Finished. Took 0.0639593521754 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509069369009\n",
      "Finished!\n",
      "Best error: 0.500204302745\n",
      "Checking...\n",
      "0 [ 0.0005799] [-1]\n",
      "1 [-0.0023572] [1]\n",
      "2 [ 0.0005926] [-1]\n",
      "3 [ 0.00059269] [-1]\n",
      "4 [ 0.00070953] [-1]\n",
      "5 [ 0.00094272] [-1]\n",
      "6 [ 0.00080975] [1]\n",
      "Accuracy: train 0.521186440678 val 0.5 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# moar experiments - try hierarchy\n",
    "\n",
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "\n",
    "C1 = nnb.PerceptronLayer(insize=2*10, outsize=10)  #, activation_func=nnb.activation.ReLU)\n",
    "qamodel = (L0 & L1) | nnb.ConcatenationModel() | C1\n",
    "\n",
    "C2 = nnb.PerceptronLayer(insize=2*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (qamodel & L2) | nnb.ConcatenationModel() | C2\n",
    "try_model(model, s0[:200], s1[:200], s2[:200], y[:200], L2_reg=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[----------] \n",
      "Finished. Took 0.0888191302617 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500957958481\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[----------] \n",
      "Finished. Took 0.0870272358259 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50158509735\n",
      "~Epoch 3~\n",
      "[----------] \n",
      "Finished. Took 0.0885104179382 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504333870123\n",
      "~Epoch 4~\n",
      "[----------] \n",
      "Finished. Took 0.0876606663068 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503840694988\n",
      "~Epoch 5~\n",
      "[----------] \n",
      "Finished. Took 0.0882599473 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505143237825\n",
      "~Epoch 6~\n",
      "[----------] \n",
      "Finished. Took 0.0875824173292 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501512641975\n",
      "~Epoch 7~\n",
      "[----------] \n",
      "Finished. Took 0.0871998349826 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515675752182\n",
      "~Epoch 8~\n",
      "[----------] \n",
      "Finished. Took 0.0874417861303 minutes.\n",
      "Evaluating...\n",
      "Error = 0.495795221328\n",
      "New best!\n",
      "~Epoch 9~\n",
      "[----------] \n",
      "Finished. Took 0.0871418992678 minutes.\n",
      "Evaluating...\n",
      "Error = 0.489819101383\n",
      "New best!\n",
      "~Epoch 10~\n",
      "[----------] \n",
      "Finished. Took 0.0882779161135 minutes.\n",
      "Evaluating...\n",
      "Error = 0.53753098396\n",
      "~Epoch 11~\n",
      "[----------] \n",
      "Finished. Took 0.0874283830325 minutes.\n",
      "Evaluating...\n",
      "Error = 0.487821228759\n",
      "New best!\n",
      "~Epoch 12~\n",
      "[----------] \n",
      "Finished. Took 0.0881759007772 minutes.\n",
      "Evaluating...\n",
      "Error = 0.512154344592\n",
      "~Epoch 13~\n",
      "[----------] \n",
      "Finished. Took 0.0877078851064 minutes.\n",
      "Evaluating...\n",
      "Error = 0.529286202406\n",
      "~Epoch 14~\n",
      "[----------] \n",
      "Finished. Took 0.0882815321287 minutes.\n",
      "Evaluating...\n",
      "Error = 0.535803551121\n",
      "~Epoch 15~\n",
      "[----------] \n",
      "Finished. Took 0.0876185536385 minutes.\n",
      "Evaluating...\n",
      "Error = 0.585004501481\n",
      "~Epoch 16~\n",
      "[----------] \n",
      "Finished. Took 0.0879242499669 minutes.\n",
      "Evaluating...\n",
      "Error = 0.649362599814\n",
      "~Epoch 17~\n",
      "[----------] \n",
      "Finished. Took 0.0887906829516 minutes.\n",
      "Evaluating...\n",
      "Error = 0.617719836258\n",
      "~Epoch 18~\n",
      "[----------] \n",
      "Finished. Took 0.0889089186986 minutes.\n",
      "Evaluating...\n",
      "Error = 0.664502570307\n",
      "~Epoch 19~\n",
      "[----------] \n",
      "Finished. Took 0.0900127331416 minutes.\n",
      "Evaluating...\n",
      "Error = 0.623351321082\n",
      "~Epoch 20~\n",
      "[----------] \n",
      "Finished. Took 0.0890651146571 minutes.\n",
      "Evaluating...\n",
      "Error = 0.557152327207\n",
      "~Epoch 21~\n",
      "[----------] \n",
      "Finished. Took 0.0892202178637 minutes.\n",
      "Evaluating...\n",
      "Error = 0.707924590485\n",
      "~Epoch 22~\n",
      "[----------] \n",
      "Finished. Took 0.090217034022 minutes.\n",
      "Evaluating...\n",
      "Error = 0.65883405687\n",
      "~Epoch 23~\n",
      "[----------] \n",
      "Finished. Took 0.089284435908 minutes.\n",
      "Evaluating...\n",
      "Error = 0.631333241419\n",
      "~Epoch 24~\n",
      "[----------] \n",
      "Finished. Took 0.0892552812894 minutes.\n",
      "Evaluating...\n",
      "Error = 0.638463590337\n",
      "~Epoch 25~\n",
      "[----------] \n",
      "Finished. Took 0.0949373165766 minutes.\n",
      "Evaluating...\n",
      "Error = 0.575777697216\n",
      "~Epoch 26~\n",
      "[----------] \n",
      "Finished. Took 0.088563366731 minutes.\n",
      "Evaluating...\n",
      "Error = 0.686644365232\n",
      "~Epoch 27~\n",
      "[----------] \n",
      "Finished. Took 0.0895174185435 minutes.\n",
      "Evaluating...\n",
      "Error = 0.692829842434\n",
      "~Epoch 28~\n",
      "[----------] \n",
      "Finished. Took 0.0881788651148 minutes.\n",
      "Evaluating...\n",
      "Error = 0.640720187303\n",
      "~Epoch 29~\n",
      "[----------] \n",
      "Finished. Took 0.0895100156466 minutes.\n",
      "Evaluating...\n",
      "Error = 0.648700704785\n",
      "~Epoch 30~\n",
      "[----------] \n",
      "Finished. Took 0.0883030811946 minutes.\n",
      "Evaluating...\n",
      "Error = 0.597849333266\n",
      "~Epoch 31~\n",
      "[----------] \n",
      "Finished. Took 0.0883086363475 minutes.\n",
      "Evaluating...\n",
      "Error = 0.686939940778\n",
      "~Epoch 32~\n",
      "[----------] \n",
      "Finished. Took 0.0887843847275 minutes.\n",
      "Evaluating...\n",
      "Error = 0.628801229772\n",
      "~Epoch 33~\n",
      "[----------] \n",
      "Finished. Took 0.0902012507121 minutes.\n",
      "Evaluating...\n",
      "Error = 0.645660561259\n",
      "~Epoch 34~\n",
      "[----------] \n",
      "Finished. Took 0.0891927838326 minutes.\n",
      "Evaluating...\n",
      "Error = 0.757139251828\n",
      "~Epoch 35~\n",
      "[----------] \n",
      "Finished. Took 0.0881246010462 minutes.\n",
      "Evaluating...\n",
      "Error = 0.765818537534\n",
      "~Epoch 36~\n",
      "[----------] \n",
      "Finished. Took 0.0894102493922 minutes.\n",
      "Evaluating...\n",
      "Error = 0.775365328229\n",
      "~Epoch 37~\n",
      "[----------] \n",
      "Finished. Took 0.0884796659152 minutes.\n",
      "Evaluating...\n",
      "Error = 0.742439619578\n",
      "~Epoch 38~\n",
      "[----------] \n",
      "Finished. Took 0.087966632843 minutes.\n",
      "Evaluating...\n",
      "Error = 0.745025193319\n",
      "~Epoch 39~\n",
      "[----------] \n",
      "Finished. Took 0.0881872971853 minutes.\n",
      "Evaluating...\n",
      "Error = 0.717345076946\n",
      "~Epoch 40~\n",
      "[----------] \n",
      "Finished. Took 0.0885720332464 minutes.\n",
      "Evaluating...\n",
      "Error = 0.684808569017\n",
      "~Epoch 41~\n",
      "[----------] \n",
      "Finished. Took 0.0896765510241 minutes.\n",
      "Evaluating...\n",
      "Error = 0.758734693691\n",
      "~Epoch 42~\n",
      "[----------] \n",
      "Finished. Took 0.0892052809397 minutes.\n",
      "Evaluating...\n",
      "Error = 0.660883337567\n",
      "~Epoch 43~\n",
      "[----------] \n",
      "Finished. Took 0.0899810989698 minutes.\n",
      "Evaluating...\n",
      "Error = 0.76079707347\n",
      "~Epoch 44~\n",
      "[----------] \n",
      "Finished. Took 0.0902749816577 minutes.\n",
      "Evaluating...\n",
      "Error = 0.71604970713\n",
      "~Epoch 45~\n",
      "[----------] \n",
      "Finished. Took 0.0885958353678 minutes.\n",
      "Evaluating...\n",
      "Error = 0.770299051298\n",
      "~Epoch 46~\n",
      "[----------] \n",
      "Finished. Took 0.0899363517761 minutes.\n",
      "Evaluating...\n",
      "Error = 0.759767638047\n",
      "~Epoch 47~\n",
      "[----------] \n",
      "Finished. Took 0.0882674336433 minutes.\n",
      "Evaluating...\n",
      "Error = 0.708172002319\n",
      "~Epoch 48~\n",
      "[----------] \n",
      "Finished. Took 0.0878605643908 minutes.\n",
      "Evaluating...\n",
      "Error = 0.710078952933\n",
      "~Epoch 49~\n",
      "[----------] \n",
      "Finished. Took 0.0894105831782 minutes.\n",
      "Evaluating...\n",
      "Error = 0.78086864715\n",
      "~Epoch 50~\n",
      "[----------] \n",
      "Finished. Took 0.0885352492332 minutes.\n",
      "Evaluating...\n",
      "Error = 0.725908131675\n",
      "~Epoch 51~\n",
      "[----------] \n",
      "Finished. Took 0.0874023358027 minutes.\n",
      "Evaluating...\n",
      "Error = 0.657848023763\n",
      "Finished!\n",
      "Best error: 0.487821228759\n",
      "Checking...\n",
      "0 [-0.84545292] [-1]\n",
      "1 [ 0.29979921] [1]\n",
      "2 [-0.171089] [-1]\n",
      "3 [-0.43346331] [-1]\n",
      "4 [-0.9164349] [-1]\n",
      "5 [-0.89145894] [-1]\n",
      "6 [-0.32916689] [1]\n",
      "Accuracy: train 0.783898305085 val 0.6 val_base 0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/Theano-0.7.0-py2.7.egg/theano/scan_module/scan_perform_ext.py:133: RuntimeWarning: numpy.ndarray size changed, may indicate binary incompatibility\n",
      "  from scan_perform.scan_perform import *\n"
     ]
    }
   ],
   "source": [
    "# moar experiments\n",
    "\n",
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, h0=np.zeros(shape=(10,), dtype=theano.config.floatX), model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, h0=np.zeros(shape=(10,), dtype=theano.config.floatX), model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, h0=np.zeros(shape=(10,), dtype=theano.config.floatX), model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model, s0[:200], s1[:200], s2[:200], y[:200], L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[----------] \n",
      "Finished. Took 0.0845589836438 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501358014788\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[----------] \n",
      "Finished. Took 0.085214749972 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499122361749\n",
      "New best!\n",
      "~Epoch 3~\n",
      "[----------] \n",
      "Finished. Took 0.0838472008705 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498413014051\n",
      "New best!\n",
      "~Epoch 4~\n",
      "[----------] \n",
      "Finished. Took 0.0840673685074 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503338874358\n",
      "~Epoch 5~\n",
      "[----------] \n",
      "Finished. Took 0.0820211172104 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497372318792\n",
      "New best!\n",
      "~Epoch 6~\n",
      "[----------] \n",
      "Finished. Took 0.0860601345698 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499694159084\n",
      "~Epoch 7~\n",
      "[----------] \n",
      "Finished. Took 0.0840774178505 minutes.\n",
      "Evaluating...\n",
      "Error = 0.529186830661\n",
      "~Epoch 8~\n",
      "[----------] \n",
      "Finished. Took 0.0852189024289 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513377143498\n",
      "~Epoch 9~\n",
      "[----------] \n",
      "Finished. Took 0.0835673809052 minutes.\n",
      "Evaluating...\n",
      "Error = 0.511034395032\n",
      "~Epoch 10~\n",
      "[----------] \n",
      "Finished. Took 0.0840396324793 minutes.\n",
      "Evaluating...\n",
      "Error = 0.513910986519\n",
      "~Epoch 11~\n",
      "[----------] \n",
      "Finished. Took 0.0835260828336 minutes.\n",
      "Evaluating...\n",
      "Error = 0.516893362173\n",
      "~Epoch 12~\n",
      "[----------] \n",
      "Finished. Took 0.0843402862549 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496697852577\n",
      "New best!\n",
      "~Epoch 13~\n",
      "[----------] \n",
      "Finished. Took 0.0834467689196 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502118085194\n",
      "~Epoch 14~\n",
      "[----------] \n",
      "Finished. Took 0.0841917157173 minutes.\n",
      "Evaluating...\n",
      "Error = 0.5153067669\n",
      "~Epoch 15~\n",
      "[----------] \n",
      "Finished. Took 0.0831967155139 minutes.\n",
      "Evaluating...\n",
      "Error = 0.528749733759\n",
      "~Epoch 16~\n",
      "[----------] \n",
      "Finished. Took 0.0843673507373 minutes.\n",
      "Evaluating...\n",
      "Error = 0.551287876461\n",
      "~Epoch 17~\n",
      "[----------] \n",
      "Finished. Took 0.0841631174088 minutes.\n",
      "Evaluating...\n",
      "Error = 0.518850289553\n",
      "~Epoch 18~\n",
      "[----------] \n",
      "Finished. Took 0.0846482316653 minutes.\n",
      "Evaluating...\n",
      "Error = 0.573369384666\n",
      "~Epoch 19~\n",
      "[----------] \n",
      "Finished. Took 0.0868474841118 minutes.\n",
      "Evaluating...\n",
      "Error = 0.53091702901\n",
      "~Epoch 20~\n",
      "[----------] \n",
      "Finished. Took 0.0867787162463 minutes.\n",
      "Evaluating...\n",
      "Error = 0.60153466329\n",
      "~Epoch 21~\n",
      "[----------] \n",
      "Finished. Took 0.088539981842 minutes.\n",
      "Evaluating...\n",
      "Error = 0.515277911859\n",
      "~Epoch 22~\n",
      "[----------] \n",
      "Finished. Took 0.0889196157455 minutes.\n",
      "Evaluating...\n",
      "Error = 0.636672478437\n",
      "~Epoch 23~\n",
      "[----------] \n",
      "Finished. Took 0.0898417512576 minutes.\n",
      "Evaluating...\n",
      "Error = 0.729162530904\n",
      "~Epoch 24~\n",
      "[----------] \n",
      "Finished. Took 0.0901450355848 minutes.\n",
      "Evaluating...\n",
      "Error = 0.623846895563\n",
      "~Epoch 25~\n",
      "[----------] \n",
      "Finished. Took 0.092857114474 minutes.\n",
      "Evaluating...\n",
      "Error = 0.698496320543\n",
      "~Epoch 26~\n",
      "[----------] \n",
      "Finished. Took 0.0876817822456 minutes.\n",
      "Evaluating...\n",
      "Error = 0.732373465847\n",
      "~Epoch 27~\n",
      "[----------] \n",
      "Finished. Took 0.0839065512021 minutes.\n",
      "Evaluating...\n",
      "Error = 0.755499440767\n",
      "~Epoch 28~\n",
      "[----------] \n",
      "Finished. Took 0.0830939809481 minutes.\n",
      "Evaluating...\n",
      "Error = 0.78981953489\n",
      "~Epoch 29~\n",
      "[----------] \n",
      "Finished. Took 0.0883158167203 minutes.\n",
      "Evaluating...\n",
      "Error = 0.679129510006\n",
      "~Epoch 30~\n",
      "[----------] \n",
      "Finished. Took 0.0870797673861 minutes.\n",
      "Evaluating...\n",
      "Error = 0.804682922273\n",
      "~Epoch 31~\n",
      "[----------] \n",
      "Finished. Took 0.0868611812592 minutes.\n",
      "Evaluating...\n",
      "Error = 0.784442138253\n",
      "~Epoch 32~\n",
      "[----------] \n",
      "Finished. Took 0.086023469766 minutes.\n",
      "Evaluating...\n",
      "Error = 0.775464585936\n",
      "~Epoch 33~\n",
      "[----------] \n",
      "Finished. Took 0.0840981523196 minutes.\n",
      "Evaluating...\n",
      "Error = 0.760805673419\n",
      "~Epoch 34~\n",
      "[----------] \n",
      "Finished. Took 0.0836679498355 minutes.\n",
      "Evaluating...\n",
      "Error = 0.84882995017\n",
      "~Epoch 35~\n",
      "[----------] \n",
      "Finished. Took 0.0840958833694 minutes.\n",
      "Evaluating...\n",
      "Error = 0.773058663099\n",
      "~Epoch 36~\n",
      "[----------] \n",
      "Finished. Took 0.0837616165479 minutes.\n",
      "Evaluating...\n",
      "Error = 0.68016136261\n",
      "~Epoch 37~\n",
      "[----------] \n",
      "Finished. Took 0.0842925667763 minutes.\n",
      "Evaluating...\n",
      "Error = 0.873555682603\n",
      "~Epoch 38~\n",
      "[----------] \n",
      "Finished. Took 0.0836460828781 minutes.\n",
      "Evaluating...\n",
      "Error = 0.893437515041\n",
      "~Epoch 39~\n",
      "[----------] \n",
      "Finished. Took 0.0862747351329 minutes.\n",
      "Evaluating...\n",
      "Error = 0.895786164728\n",
      "~Epoch 40~\n",
      "[----------] \n",
      "Finished. Took 0.084677751859 minutes.\n",
      "Evaluating...\n",
      "Error = 0.828187588018\n",
      "~Epoch 41~\n",
      "[----------] \n",
      "Finished. Took 0.0849760174751 minutes.\n",
      "Evaluating...\n",
      "Error = 0.786189578228\n",
      "~Epoch 42~\n",
      "[----------] \n",
      "Finished. Took 0.0907589038213 minutes.\n",
      "Evaluating...\n",
      "Error = 0.818217729819\n",
      "~Epoch 43~\n",
      "[----------] \n",
      "Finished. Took 0.0885839184125 minutes.\n",
      "Evaluating...\n",
      "Error = 0.876503465673\n",
      "~Epoch 44~\n",
      "[----------] \n",
      "Finished. Took 0.0891054352125 minutes.\n",
      "Evaluating...\n",
      "Error = 0.894827768164\n",
      "~Epoch 45~\n",
      "[----------] \n",
      "Finished. Took 0.0853153149287 minutes.\n",
      "Evaluating...\n",
      "Error = 0.717912823448\n",
      "~Epoch 46~\n",
      "[----------] \n",
      "Finished. Took 0.0844216982524 minutes.\n",
      "Evaluating...\n",
      "Error = 0.83479353266\n",
      "~Epoch 47~\n",
      "[----------] \n",
      "Finished. Took 0.0853721499443 minutes.\n",
      "Evaluating...\n",
      "Error = 0.908505694955\n",
      "~Epoch 48~\n",
      "[----------] \n",
      "Finished. Took 0.0884340365728 minutes.\n",
      "Evaluating...\n",
      "Error = 0.912132571555\n",
      "~Epoch 49~\n",
      "[----------] \n",
      "Finished. Took 0.0896174867948 minutes.\n",
      "Evaluating...\n",
      "Error = 0.863745081854\n",
      "~Epoch 50~\n",
      "[----------] \n",
      "Finished. Took 0.0900812149048 minutes.\n",
      "Evaluating...\n",
      "Error = 0.914620715387\n",
      "~Epoch 51~\n",
      "[----------] \n",
      "Finished. Took 0.0896432518959 minutes.\n",
      "Evaluating...\n",
      "Error = 0.889432233976\n",
      "~Epoch 52~\n",
      "[----------] \n",
      "Finished. Took 0.0855853676796 minutes.\n",
      "Evaluating...\n",
      "Error = 0.802259311319\n",
      "Finished!\n",
      "Best error: 0.496697852577\n",
      "Checking...\n",
      "0 [-0.67457136] [-1]\n",
      "1 [ 0.08242677] [1]\n",
      "2 [ 0.3172713] [-1]\n",
      "3 [-0.41661634] [-1]\n",
      "4 [-0.75236761] [-1]\n",
      "5 [-0.72149042] [-1]\n",
      "6 [-0.09718985] [1]\n",
      "Accuracy: train 0.792372881356 val 0.55 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "# moar experiments\n",
    "\n",
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model, s0[:200], s1[:200], s2[:200], y[:200], L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN Models (full dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 1.60305870374 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500098191283\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 1.59851053158 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500203903116\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 1.61304879983 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500005266662\n",
      "New best!\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 1.59925789833 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500144910107\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 1.61563773155 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500183155707\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 1.59077801704 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500062925921\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 1.60953396559 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499974214464\n",
      "New best!\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 1.58993933598 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500243055911\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 1.60596858263 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504467441184\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 1.58714205027 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50009035526\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 1.60218755007 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500428071475\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 1.58903076649 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500153715169\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 1.59558986823 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499945997402\n",
      "New best!\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 1.5847178181 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500270266198\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 1.5991928498 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50139547587\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 1.59348481496 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499961534442\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 1.59689329863 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500028248364\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 1.58521181742 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500131739425\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 1.59848686854 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500042034754\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 1.57670763334 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50124867609\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 1.59499901533 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500034495451\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 1.58273703257 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499985781759\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 1.595523417 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500151898127\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 1.5704961141 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500306158981\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 1.59708646536 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499898950993\n",
      "New best!\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 1.58144428333 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500191890673\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 1.59975515207 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499949037258\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 1.59488438368 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499899847503\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 1.60715348323 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499904836225\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 1.60072633028 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499934813353\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 1.6116367499 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503429407141\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 1.60008765062 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500296952577\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 1.60218679905 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49984759512\n",
      "New best!\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 1.59262788296 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500429237232\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 1.59730816682 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500207733781\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 1.58930324713 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500044514413\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 1.59458829959 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500262750858\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 1.5869914492 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500025664532\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 1.59674686988 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499891957793\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 1.59068368276 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500451855638\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 1.78181728125 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500124908989\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 1.58460318247 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499966270553\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 1.60404229959 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49997715552\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 1.59195893208 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499760601269\n",
      "New best!\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 1.5981889685 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500620267919\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 1.5870976011 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499788954895\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 1.60150238673 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499865833628\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 1.59236471653 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499875705595\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 1.60570954879 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499887362701\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 1.58513498306 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499995653684\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 1.60111906528 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499656027874\n",
      "New best!\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 1.58023430109 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500676748777\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 1.59873213371 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499663027188\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 1.60152798096 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500191158572\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 1.59780245225 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50317136745\n",
      "~Epoch 56~\n",
      "[--------- ] \n",
      "Finished. Took 1.59568908215 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499833753831\n",
      "~Epoch 57~\n",
      "[--------- ] \n",
      "Finished. Took 1.60268669923 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50046729912\n",
      "~Epoch 58~\n",
      "[--------- ] \n",
      "Finished. Took 1.61063123544 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499538925022\n",
      "New best!\n",
      "~Epoch 59~\n",
      "[--------- ] \n",
      "Finished. Took 1.60631283522 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500031543151\n",
      "~Epoch 60~\n",
      "[--------- ] \n",
      "Finished. Took 1.60338308414 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499536305929\n",
      "New best!\n",
      "~Epoch 61~\n",
      "[--------- ] \n",
      "Finished. Took 1.606526951 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49995224194\n",
      "~Epoch 62~\n",
      "[--------- ] \n",
      "Finished. Took 1.60386393468 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499549953257\n",
      "~Epoch 63~\n",
      "[--------- ] \n",
      "Finished. Took 1.60003294945 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499764311352\n",
      "~Epoch 64~\n",
      "[--------- ] \n",
      "Finished. Took 1.60350961685 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499429120277\n",
      "New best!\n",
      "~Epoch 65~\n",
      "[--------- ] \n",
      "Finished. Took 1.60294603507 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499411216458\n",
      "New best!\n",
      "~Epoch 66~\n",
      "[--------- ] \n",
      "Finished. Took 1.59674990177 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49966901006\n",
      "~Epoch 67~\n",
      "[--------- ] \n",
      "Finished. Took 1.59608833392 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499464966039\n",
      "~Epoch 68~\n",
      "[--------- ] \n",
      "Finished. Took 1.59490481615 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499451975125\n",
      "~Epoch 69~\n",
      "[--------- ] \n",
      "Finished. Took 1.59433780114 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499514870812\n",
      "~Epoch 70~\n",
      "[--------- ] \n",
      "Finished. Took 1.59872906605 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500894771349\n",
      "~Epoch 71~\n",
      "[--------- ] \n",
      "Finished. Took 1.59720342954 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500798069338\n",
      "~Epoch 72~\n",
      "[--------- ] \n",
      "Finished. Took 1.59751033386 minutes.\n",
      "Evaluating...\n",
      "Error = 0.4994568131\n",
      "~Epoch 73~\n",
      "[--------- ] \n",
      "Finished. Took 1.59765398105 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499267988892\n",
      "New best!\n",
      "~Epoch 74~\n",
      "[--------- ] \n",
      "Finished. Took 1.59259883165 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499394640131\n",
      "~Epoch 75~\n",
      "[--------- ] \n",
      "Finished. Took 1.60015970071 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49924532516\n",
      "New best!\n",
      "~Epoch 76~\n",
      "[--------- ] \n",
      "Finished. Took 1.59522803227 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499494181586\n",
      "~Epoch 77~\n",
      "[--------- ] \n",
      "Finished. Took 1.60206548373 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499883849898\n",
      "~Epoch 78~\n",
      "[--------- ] \n",
      "Finished. Took 1.60000426769 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499269276997\n",
      "~Epoch 79~\n",
      "[--------- ] \n",
      "Finished. Took 1.59928249915 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500804921116\n",
      "~Epoch 80~\n",
      "[--------- ] \n",
      "Finished. Took 1.59892743031 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49917343613\n",
      "New best!\n",
      "~Epoch 81~\n",
      "[--------- ] \n",
      "Finished. Took 1.59754793247 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499263257159\n",
      "~Epoch 82~\n",
      "[--------- ] \n",
      "Finished. Took 1.59854396582 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499165646391\n",
      "New best!\n",
      "~Epoch 83~\n",
      "[--------- ] \n",
      "Finished. Took 1.59138978322 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499203176495\n",
      "~Epoch 84~\n",
      "[--------- ] \n",
      "Finished. Took 1.5969127655 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49963643685\n",
      "~Epoch 85~\n",
      "[--------- ] \n",
      "Finished. Took 1.59531498353 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499347763835\n",
      "~Epoch 86~\n",
      "[--------- ] \n",
      "Finished. Took 1.60005731583 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499225496499\n",
      "~Epoch 87~\n",
      "[--------- ] \n",
      "Finished. Took 1.59444083373 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499253673178\n",
      "~Epoch 88~\n",
      "[--------- ] \n",
      "Finished. Took 1.59675086339 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500288873726\n",
      "~Epoch 89~\n",
      "[--------- ] \n",
      "Finished. Took 1.59632833401 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499174118005\n",
      "~Epoch 90~\n",
      "[--------- ] \n",
      "Finished. Took 1.60239344835 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499556325737\n",
      "~Epoch 91~\n",
      "[--------- ] \n",
      "Finished. Took 1.59957198302 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49928898455\n",
      "~Epoch 92~\n",
      "[--------- ] \n",
      "Finished. Took 1.58885401487 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500397169437\n",
      "~Epoch 93~\n",
      "[--------- ] \n",
      "Finished. Took 1.59455313285 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499674322617\n",
      "~Epoch 94~\n",
      "[--------- ] \n",
      "Finished. Took 1.59326896667 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499247668292\n",
      "~Epoch 95~\n",
      "[--------- ] \n",
      "Finished. Took 1.59487021367 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49925015632\n",
      "~Epoch 96~\n",
      "[--------- ] \n",
      "Finished. Took 1.59558800062 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500270574623\n",
      "~Epoch 97~\n",
      "[--------- ] \n",
      "Finished. Took 1.59082603455 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499700683305\n",
      "~Epoch 98~\n",
      "[--------- ] \n",
      "Finished. Took 1.59298181534 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49935849659\n",
      "~Epoch 99~\n",
      "[--------- ] \n",
      "Finished. Took 1.59074894985 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499387378958\n",
      "~Epoch 100~\n",
      "[--------- ] \n",
      "Finished. Took 1.59411751429 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49945232917\n",
      "~Epoch 101~\n",
      "[--------- ] \n",
      "Finished. Took 1.59808511734 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499577144988\n",
      "~Epoch 102~\n",
      "[--------- ] \n",
      "Finished. Took 1.60414886475 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499797273489\n",
      "~Epoch 103~\n",
      "[--------- ] \n",
      "Finished. Took 1.59834390084 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499670481115\n",
      "~Epoch 104~\n",
      "[--------- ] \n",
      "Finished. Took 1.61268300215 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499505951665\n",
      "~Epoch 105~\n",
      "[--------- ] \n",
      "Finished. Took 1.60572783152 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501437643043\n",
      "~Epoch 106~\n",
      "[--------- ] \n",
      "Finished. Took 1.6053940018 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501308535144\n",
      "~Epoch 107~\n",
      "[--------- ] \n",
      "Finished. Took 1.60434378386 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499679276265\n",
      "~Epoch 108~\n",
      "[--------- ] \n",
      "Finished. Took 1.79635104736 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50031625236\n",
      "~Epoch 109~\n",
      "[--------- ] \n",
      "Finished. Took 1.5983103315 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49978943573\n",
      "~Epoch 110~\n",
      "[--------- ] \n",
      "Finished. Took 1.60449086825 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500307764961\n",
      "~Epoch 111~\n",
      "[--------- ] \n",
      "Finished. Took 1.60772106647 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499825855003\n",
      "~Epoch 112~\n",
      "[--------- ] \n",
      "Finished. Took 1.60718520085 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500889497388\n",
      "~Epoch 113~\n",
      "[--------- ] \n",
      "Finished. Took 1.59695438544 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499940181536\n",
      "~Epoch 114~\n",
      "[--------- ] \n",
      "Finished. Took 1.61097979943 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500526199658\n",
      "~Epoch 115~\n",
      "[--------- ] \n",
      "Finished. Took 1.60704346498 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501929137658\n",
      "~Epoch 116~\n",
      "[--------- ] \n",
      "Finished. Took 1.60917031765 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500293493411\n",
      "~Epoch 117~\n",
      "[--------- ] \n",
      "Finished. Took 1.61082393328 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500082820747\n",
      "~Epoch 118~\n",
      "[--------- ] \n",
      "Finished. Took 1.60461903413 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501842610302\n",
      "~Epoch 119~\n",
      "[--------- ] \n",
      "Finished. Took 1.6080175519 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50089107465\n",
      "~Epoch 120~\n",
      "[--------- ] \n",
      "Finished. Took 1.60591498216 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500940262253\n",
      "~Epoch 121~\n",
      "[--------- ] \n",
      "Finished. Took 1.60538824797 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501319076419\n",
      "~Epoch 122~\n",
      "[--------- ] \n",
      "Finished. Took 1.62334725062 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502140042731\n",
      "Finished!\n",
      "Best error: 0.499165646391\n",
      "Checking...\n",
      "0 [-0.0403889] [-1]\n",
      "1 [ 0.05077369] [1]\n",
      "2 [-0.03426339] [-1]\n",
      "3 [-0.00993044] [-1]\n",
      "4 [ 0.00425702] [-1]\n",
      "5 [ 0.01680285] [-1]\n",
      "6 [ 0.0549997] [1]\n",
      "Accuracy: train 0.544308231173 val 0.50985915493 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model0 = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model0, s0, s1, s2, y, L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 1.78559484879 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500753404726\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 1.72798031569 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504978694047\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 1.74937035243 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50093175606\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 1.71746879816 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501043002227\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 1.74941581488 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503631694834\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 1.75233989954 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500178112427\n",
      "New best!\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 1.7777565519 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50003018007\n",
      "New best!\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 1.73767441511 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500282255564\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 1.77154073318 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500432454176\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 1.75448156993 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500249248256\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 1.77363768419 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502188686042\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 1.73880585035 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502671684387\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 1.80046908061 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501173610026\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 1.75089066823 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502435825974\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 1.7835670352 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500148718834\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 1.72498603662 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500591459622\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 1.78284460306 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50327809558\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 1.73486564954 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500116200735\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 1.78858533303 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500570423983\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 1.76353654861 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500048949348\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 1.78102098306 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500155421418\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 1.75206759771 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500767960621\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 1.78034966787 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500511446854\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 1.74677658081 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500054264997\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 1.78732968569 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501647781772\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 1.76551308235 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500044505297\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 1.78169354995 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500127940506\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 1.74940645297 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501385809279\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 1.79151301384 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501048005681\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 1.77161338329 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500011686525\n",
      "New best!\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 1.79095528523 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500257016885\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 1.78735763232 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500117812885\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 1.79232379993 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505767086965\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 1.7816751798 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503169922566\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 1.79592321714 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500718076738\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 1.78623268604 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503111208374\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 1.79211676518 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504399347919\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 1.78159933488 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501667946886\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 1.80450750192 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499872054403\n",
      "New best!\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 1.78700679938 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499869831609\n",
      "New best!\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 1.78993620078 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501078246471\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 1.77331813176 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50144735915\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 1.78958521684 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50082675286\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 1.76256216764 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501909250354\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 1.7994773825 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499877679457\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 1.77163663308 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502002098792\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 1.97699985107 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500441571578\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 1.75125536521 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500786783127\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 1.79824405114 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501214560223\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 1.76482936541 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499718378084\n",
      "New best!\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 1.79337478479 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500253368173\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 1.77743655046 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499732353772\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 1.78909635146 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501091583515\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 1.77394386927 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499810600724\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 1.78241750002 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502509511176\n",
      "~Epoch 56~\n",
      "[--------- ] \n",
      "Finished. Took 1.76613719861 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49978437849\n",
      "~Epoch 57~\n",
      "[--------- ] \n",
      "Finished. Took 1.80139438311 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499979279726\n",
      "~Epoch 58~\n",
      "[--------- ] \n",
      "Finished. Took 1.78475910028 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500210253916\n",
      "~Epoch 59~\n",
      "[--------- ] \n",
      "Finished. Took 1.79779700041 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500842320508\n",
      "~Epoch 60~\n",
      "[--------- ] \n",
      "Finished. Took 1.77565876643 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499750496156\n",
      "~Epoch 61~\n",
      "[--------- ] \n",
      "Finished. Took 1.79589001338 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499735471293\n",
      "~Epoch 62~\n",
      "[--------- ] \n",
      "Finished. Took 1.78098194997 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500222535317\n",
      "~Epoch 63~\n",
      "[--------- ] \n",
      "Finished. Took 1.79056394895 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501022864003\n",
      "~Epoch 64~\n",
      "[--------- ] \n",
      "Finished. Took 1.77832913001 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500137728873\n",
      "~Epoch 65~\n",
      "[--------- ] \n",
      "Finished. Took 1.78658301433 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505999796507\n",
      "~Epoch 66~\n",
      "[--------- ] \n",
      "Finished. Took 1.77929061651 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499433689185\n",
      "New best!\n",
      "~Epoch 67~\n",
      "[--------- ] \n",
      "Finished. Took 1.78221361637 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49953610063\n",
      "~Epoch 68~\n",
      "[--------- ] \n",
      "Finished. Took 1.78154711723 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499744725045\n",
      "~Epoch 69~\n",
      "[--------- ] \n",
      "Finished. Took 1.78349351486 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499353186276\n",
      "New best!\n",
      "~Epoch 70~\n",
      "[--------- ] \n",
      "Finished. Took 1.79108169874 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502732502332\n",
      "~Epoch 71~\n",
      "[--------- ] \n",
      "Finished. Took 1.79979298512 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499327512841\n",
      "New best!\n",
      "~Epoch 72~\n",
      "[--------- ] \n",
      "Finished. Took 1.78098743359 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503455799166\n",
      "~Epoch 73~\n",
      "[--------- ] \n",
      "Finished. Took 1.79055066506 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499483051963\n",
      "~Epoch 74~\n",
      "[--------- ] \n",
      "Finished. Took 1.77951878309 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502644496438\n",
      "~Epoch 75~\n",
      "[--------- ] \n",
      "Finished. Took 1.80734045108 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500372145165\n",
      "~Epoch 76~\n",
      "[--------- ] \n",
      "Finished. Took 1.77732836803 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499770037629\n",
      "~Epoch 77~\n",
      "[--------- ] \n",
      "Finished. Took 1.79510783354 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500613334754\n",
      "~Epoch 78~\n",
      "[--------- ] \n",
      "Finished. Took 1.78688110113 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499575564316\n",
      "~Epoch 79~\n",
      "[--------- ] \n",
      "Finished. Took 1.79454251528 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501780056039\n",
      "~Epoch 80~\n",
      "[--------- ] \n",
      "Finished. Took 1.77627168099 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499723165286\n",
      "~Epoch 81~\n",
      "[--------- ] \n",
      "Finished. Took 1.78802280029 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499755676607\n",
      "~Epoch 82~\n",
      "[--------- ] \n",
      "Finished. Took 1.78335868518 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499429866618\n",
      "~Epoch 83~\n",
      "[--------- ] \n",
      "Finished. Took 1.79220593373 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499408421763\n",
      "~Epoch 84~\n",
      "[--------- ] \n",
      "Finished. Took 1.78164576689 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49892435442\n",
      "New best!\n",
      "~Epoch 85~\n",
      "[--------- ] \n",
      "Finished. Took 1.79025719961 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500065822769\n",
      "~Epoch 86~\n",
      "[--------- ] \n",
      "Finished. Took 1.76945968072 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49896315187\n",
      "~Epoch 87~\n",
      "[--------- ] \n",
      "Finished. Took 1.78588569959 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501293936574\n",
      "~Epoch 88~\n",
      "[--------- ] \n",
      "Finished. Took 1.7856959343 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498906828109\n",
      "New best!\n",
      "~Epoch 89~\n",
      "[--------- ] \n",
      "Finished. Took 1.79025270144 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499180002015\n",
      "~Epoch 90~\n",
      "[--------- ] \n",
      "Finished. Took 1.77595449686 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49994324477\n",
      "~Epoch 91~\n",
      "[--------- ] \n",
      "Finished. Took 1.78146479925 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498921623614\n",
      "~Epoch 92~\n",
      "[--------- ] \n",
      "Finished. Took 1.76691809893 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498853046278\n",
      "New best!\n",
      "~Epoch 93~\n",
      "[--------- ] \n",
      "Finished. Took 1.78961290121 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501402777761\n",
      "~Epoch 94~\n",
      "[--------- ] \n",
      "Finished. Took 1.77879666487 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499724432872\n",
      "~Epoch 95~\n",
      "[--------- ] \n",
      "Finished. Took 1.7843966802 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498880864228\n",
      "~Epoch 96~\n",
      "[--------- ] \n",
      "Finished. Took 1.78298926751 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498849556086\n",
      "New best!\n",
      "~Epoch 97~\n",
      "[--------- ] \n",
      "Finished. Took 1.76296813488 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500417750808\n",
      "~Epoch 98~\n",
      "[--------- ] \n",
      "Finished. Took 1.7766432484 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500184487431\n",
      "~Epoch 99~\n",
      "[--------- ] \n",
      "Finished. Took 1.76014655034 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498633818943\n",
      "New best!\n",
      "~Epoch 100~\n",
      "[--------- ] \n",
      "Finished. Took 1.78693368435 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50304602068\n",
      "~Epoch 101~\n",
      "[--------- ] \n",
      "Finished. Took 1.7663373669 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498952215791\n",
      "~Epoch 102~\n",
      "[--------- ] \n",
      "Finished. Took 1.79247175058 minutes.\n",
      "Evaluating...\n",
      "Error = 0.5008004305\n",
      "~Epoch 103~\n",
      "[--------- ] \n",
      "Finished. Took 1.76453746557 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499906288027\n",
      "~Epoch 104~\n",
      "[--------- ] \n",
      "Finished. Took 1.79107439915 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500028161923\n",
      "~Epoch 105~\n",
      "[--------- ] \n",
      "Finished. Took 1.77223098675 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498630326549\n",
      "New best!\n",
      "~Epoch 106~\n",
      "[--------- ] \n",
      "Finished. Took 1.77662228346 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508751456919\n",
      "~Epoch 107~\n",
      "[--------- ] \n",
      "Finished. Took 1.96260333459 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498255580156\n",
      "New best!\n",
      "~Epoch 108~\n",
      "[--------- ] \n",
      "Finished. Took 1.78854748408 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502095035923\n",
      "~Epoch 109~\n",
      "[--------- ] \n",
      "Finished. Took 1.78240841627 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498435987295\n",
      "~Epoch 110~\n",
      "[--------- ] \n",
      "Finished. Took 1.78370023171 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500298492314\n",
      "~Epoch 111~\n",
      "[--------- ] \n",
      "Finished. Took 1.76378316482 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501739953309\n",
      "~Epoch 112~\n",
      "[--------- ] \n",
      "Finished. Took 1.79132574797 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499917065939\n",
      "~Epoch 113~\n",
      "[--------- ] \n",
      "Finished. Took 1.76940450271 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500074812109\n",
      "~Epoch 114~\n",
      "[--------- ] \n",
      "Finished. Took 1.7922587355 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498138760101\n",
      "New best!\n",
      "~Epoch 115~\n",
      "[--------- ] \n",
      "Finished. Took 1.77358813286 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498418368977\n",
      "~Epoch 116~\n",
      "[--------- ] \n",
      "Finished. Took 1.78117575248 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498651726414\n",
      "~Epoch 117~\n",
      "[--------- ] \n",
      "Finished. Took 1.77874693473 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500857265854\n",
      "~Epoch 118~\n",
      "[--------- ] \n",
      "Finished. Took 1.78844679991 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499525200099\n",
      "~Epoch 119~\n",
      "[--------- ] \n",
      "Finished. Took 1.78162994782 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503058742318\n",
      "~Epoch 120~\n",
      "[--------- ] \n",
      "Finished. Took 1.79080034892 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498965383467\n",
      "~Epoch 121~\n",
      "[--------- ] \n",
      "Finished. Took 1.7839451313 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498805963073\n",
      "~Epoch 122~\n",
      "[--------- ] \n",
      "Finished. Took 1.79709062974 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498075671032\n",
      "New best!\n",
      "~Epoch 123~\n",
      "[--------- ] \n",
      "Finished. Took 1.78875851631 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500016640907\n",
      "~Epoch 124~\n",
      "[--------- ] \n",
      "Finished. Took 1.79716384808 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498479542062\n",
      "~Epoch 125~\n",
      "[--------- ] \n",
      "Finished. Took 1.79027076562 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507230717395\n",
      "~Epoch 126~\n",
      "[--------- ] \n",
      "Finished. Took 1.77822850148 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499490415365\n",
      "~Epoch 127~\n",
      "[--------- ] \n",
      "Finished. Took 1.78468294938 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499630527723\n",
      "~Epoch 128~\n",
      "[--------- ] \n",
      "Finished. Took 1.79173934857 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49803123504\n",
      "New best!\n",
      "~Epoch 129~\n",
      "[--------- ] \n",
      "Finished. Took 1.79089806477 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49793707539\n",
      "New best!\n",
      "~Epoch 130~\n",
      "[--------- ] \n",
      "Finished. Took 1.78832883437 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499191538271\n",
      "~Epoch 131~\n",
      "[--------- ] \n",
      "Finished. Took 1.79321691593 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499132838555\n",
      "~Epoch 132~\n",
      "[--------- ] \n",
      "Finished. Took 1.79463186661 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49787077401\n",
      "New best!\n",
      "~Epoch 133~\n",
      "[--------- ] \n",
      "Finished. Took 1.79290898244 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49792363035\n",
      "~Epoch 134~\n",
      "[--------- ] \n",
      "Finished. Took 1.78861758312 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497882847213\n",
      "~Epoch 135~\n",
      "[--------- ] \n",
      "Finished. Took 1.78555043538 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498224211632\n",
      "~Epoch 136~\n",
      "[--------- ] \n",
      "Finished. Took 1.77803969781 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50056057482\n",
      "~Epoch 137~\n",
      "[--------- ] \n",
      "Finished. Took 1.78142946561 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499194029828\n",
      "~Epoch 138~\n",
      "[--------- ] \n",
      "Finished. Took 1.77542775075 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498882641646\n",
      "~Epoch 139~\n",
      "[--------- ] \n",
      "Finished. Took 1.77351305087 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499188525254\n",
      "~Epoch 140~\n",
      "[--------- ] \n",
      "Finished. Took 1.77575761477 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501004846532\n",
      "~Epoch 141~\n",
      "[--------- ] \n",
      "Finished. Took 1.79511496623 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500188040447\n",
      "~Epoch 142~\n",
      "[--------- ] \n",
      "Finished. Took 1.77994046609 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501650423194\n",
      "~Epoch 143~\n",
      "[--------- ] \n",
      "Finished. Took 1.78100736936 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50457803535\n",
      "~Epoch 144~\n",
      "[--------- ] \n",
      "Finished. Took 1.78121121724 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497963285287\n",
      "~Epoch 145~\n",
      "[--------- ] \n",
      "Finished. Took 1.78413158655 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499315476445\n",
      "~Epoch 146~\n",
      "[--------- ] \n",
      "Finished. Took 1.78544233243 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498191230092\n",
      "~Epoch 147~\n",
      "[--------- ] \n",
      "Finished. Took 1.78220236699 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498596458554\n",
      "~Epoch 148~\n",
      "[--------- ] \n",
      "Finished. Took 1.77868843079 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498637980046\n",
      "~Epoch 149~\n",
      "[--------- ] \n",
      "Finished. Took 1.78050510089 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503272852109\n",
      "~Epoch 150~\n",
      "[--------- ] \n",
      "Finished. Took 1.78668358326 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499627848594\n",
      "~Epoch 151~\n",
      "[--------- ] \n",
      "Finished. Took 1.77966923316 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501260819583\n",
      "~Epoch 152~\n",
      "[--------- ] \n",
      "Finished. Took 1.77528524796 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499856678555\n",
      "~Epoch 153~\n",
      "[--------- ] \n",
      "Finished. Took 1.78101586898 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499642896952\n",
      "~Epoch 154~\n",
      "[--------- ] \n",
      "Finished. Took 1.7794271191 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499484620646\n",
      "~Epoch 155~\n",
      "[--------- ] \n",
      "Finished. Took 1.78718378544 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49825475789\n",
      "~Epoch 156~\n",
      "[--------- ] \n",
      "Finished. Took 1.77706141472 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497975615584\n",
      "~Epoch 157~\n",
      "[--------- ] \n",
      "Finished. Took 1.7766473492 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499570643928\n",
      "~Epoch 158~\n",
      "[--------- ] \n",
      "Finished. Took 1.78551894824 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498477479759\n",
      "~Epoch 159~\n",
      "[--------- ] \n",
      "Finished. Took 1.77150966724 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499058066037\n",
      "~Epoch 160~\n",
      "[--------- ] \n",
      "Finished. Took 1.81910171509 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502277176157\n",
      "~Epoch 161~\n",
      "[--------- ] \n",
      "Finished. Took 1.78331735134 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50011659624\n",
      "~Epoch 162~\n",
      "[--------- ] \n",
      "Finished. Took 1.78861209949 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501702533095\n",
      "~Epoch 163~\n",
      "[--------- ] \n",
      "Finished. Took 1.80548871358 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498390040197\n",
      "~Epoch 164~\n",
      "[--------- ] \n",
      "Finished. Took 1.78630238374 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500189059812\n",
      "~Epoch 165~\n",
      "[--------- ] \n",
      "Finished. Took 1.78272703489 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499736759038\n",
      "~Epoch 166~\n",
      "[--------- ] \n",
      "Finished. Took 1.78835384846 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498240023859\n",
      "~Epoch 167~\n",
      "[--------- ] \n",
      "Finished. Took 1.78546348015 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498223170881\n",
      "~Epoch 168~\n",
      "[--------- ] \n",
      "Finished. Took 1.98255665302 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498680889309\n",
      "~Epoch 169~\n",
      "[--------- ] \n",
      "Finished. Took 1.78951243162 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498055002245\n",
      "~Epoch 170~\n",
      "[--------- ] \n",
      "Finished. Took 1.79157080253 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501907262971\n",
      "~Epoch 171~\n",
      "[--------- ] \n",
      "Finished. Took 1.78778649966 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498520568433\n",
      "~Epoch 172~\n",
      "[--------- ] \n",
      "Finished. Took 1.79142089685 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498399118997\n",
      "Finished!\n",
      "Best error: 0.49787077401\n",
      "Checking...\n",
      "0 [-0.0128919] [-1]\n",
      "1 [ 0.07778811] [1]\n",
      "2 [-0.10454877] [-1]\n",
      "3 [ 0.02883072] [-1]\n",
      "4 [ 0.06996683] [-1]\n",
      "5 [ 0.02067657] [-1]\n",
      "6 [ 0.15117761] [1]\n",
      "Accuracy: train 0.536952714536 val 0.518309859155 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=20)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=20)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=20)[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*20, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model1 = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model1, s0, s1, s2, y, L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 2.2949358503 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500074020595\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 2.11264136632 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500076908437\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 2.04707011779 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500184669482\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 2.03230413198 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499987650091\n",
      "New best!\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 2.02816118399 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499945002469\n",
      "New best!\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 2.16420128345 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499915157177\n",
      "New best!\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 2.30481009881 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499886588115\n",
      "New best!\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 2.54897553523 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499830438283\n",
      "New best!\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 2.10396888256 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499784531316\n",
      "New best!\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 2.31629316409 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499704002686\n",
      "New best!\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 2.27412663301 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499685353891\n",
      "New best!\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 2.06090625127 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499551985342\n",
      "New best!\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 2.02741090059 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499479486981\n",
      "New best!\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 2.09312373002 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499385848964\n",
      "New best!\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 2.72769560019 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499101450019\n",
      "New best!\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 2.04905306896 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498931088613\n",
      "New best!\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 2.15781305234 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498765090036\n",
      "New best!\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 2.67321244876 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498587910575\n",
      "New best!\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 2.79000920057 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498288862716\n",
      "New best!\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 2.1791980346 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498088082393\n",
      "New best!\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 2.07449391683 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49800190103\n",
      "New best!\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 2.0662095348 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497581335869\n",
      "New best!\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 2.1156218489 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497297578117\n",
      "New best!\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 2.06131190062 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497103165647\n",
      "New best!\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 2.07626165152 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496944417941\n",
      "New best!\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 2.070903186 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496784982366\n",
      "New best!\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 2.05689641635 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496592793495\n",
      "New best!\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 2.06024608215 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496420178061\n",
      "New best!\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 2.07265876929 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496279932267\n",
      "New best!\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 2.08380818367 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496287215641\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 2.08673871358 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496062160848\n",
      "New best!\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 2.08336448272 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496351998778\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 2.07848583062 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496139105732\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 2.08225061496 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496680230553\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 2.11399857998 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496211643402\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 2.36219968001 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496230254616\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 2.10076063077 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496764451617\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 3.00731554826 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496399415945\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 3.8204663833 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497685762684\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 3.54703496695 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496583145513\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 3.44205650091 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497235025594\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 3.32031856378 minutes.\n",
      "Evaluating...\n",
      "Error = 0.496780436703\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 3.4200742205 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497007985961\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 3.37606931527 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497174056313\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 3.28116474946 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497227802065\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 3.15441763401 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497284638484\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 3.30958185196 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497682080656\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 3.30744861762 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497266177528\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 3.17351163626 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497546148831\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 3.30429335038 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497730382729\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 3.27443424861 minutes.\n",
      "Evaluating...\n",
      "Error = 0.497964371768\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 2.95584951639 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498048956133\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 3.16892701785 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498291069401\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 3.3757901152 minutes.\n",
      "Evaluating...\n",
      "Error = 0.5015182091\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 3.27752836943 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498476415963\n",
      "~Epoch 56~\n",
      "[--------- ] \n",
      "Finished. Took 3.25086541573 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498872572502\n",
      "~Epoch 57~\n",
      "[--------- ] \n",
      "Finished. Took 3.40007445017 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498684492035\n",
      "~Epoch 58~\n",
      "[--------- ] \n",
      "Finished. Took 3.27973790169 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498703207863\n",
      "~Epoch 59~\n",
      "[--------- ] \n",
      "Finished. Took 3.25688191652 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499948738879\n",
      "~Epoch 60~\n",
      "[--------- ] \n",
      "Finished. Took 3.30412148237 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498825939449\n",
      "~Epoch 61~\n",
      "[--------- ] \n",
      "Finished. Took 3.24951161941 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499853956094\n",
      "~Epoch 62~\n",
      "[--------- ] \n",
      "Finished. Took 3.27779266834 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499044068637\n",
      "~Epoch 63~\n",
      "[--------- ] \n",
      "Finished. Took 3.13207231363 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499589066015\n",
      "~Epoch 64~\n",
      "[--------- ] \n",
      "Finished. Took 3.1188662529 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500322614196\n",
      "~Epoch 65~\n",
      "[--------- ] \n",
      "Finished. Took 2.95188819965 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501545080784\n",
      "~Epoch 66~\n",
      "[--------- ] \n",
      "Finished. Took 3.07789588372 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50016084911\n",
      "~Epoch 67~\n",
      "[--------- ] \n",
      "Finished. Took 3.14606215159 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500009803866\n",
      "~Epoch 68~\n",
      "[--------- ] \n",
      "Finished. Took 3.10478356679 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503649861395\n",
      "~Epoch 69~\n",
      "[--------- ] \n",
      "Finished. Took 2.17632674774 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500482434481\n",
      "~Epoch 70~\n",
      "[--------- ] \n",
      "Finished. Took 2.16710883379 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500042061636\n",
      "~Epoch 71~\n",
      "[--------- ] \n",
      "Finished. Took 2.40840676626 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500334235348\n",
      "Finished!\n",
      "Best error: 0.496062160848\n",
      "Checking...\n",
      "0 [ 0.05224031] [-1]\n",
      "1 [ 0.1554261] [1]\n",
      "2 [-0.04199234] [-1]\n",
      "3 [-0.01066265] [-1]\n",
      "4 [ 0.04060266] [-1]\n",
      "5 [ 0.05364958] [-1]\n",
      "6 [ 0.12067958] [1]\n",
      "Accuracy: train 0.552714535902 val 0.533098591549 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model2 = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model2, s0, s1, s2, y, L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 2.29777733485 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49997147047\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 2.29882065058 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500004758752\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 2.26569316785 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499930540751\n",
      "New best!\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 2.25102778276 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499906717101\n",
      "New best!\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 2.27970138391 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499873056762\n",
      "New best!\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 2.24660113255 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499843993268\n",
      "New best!\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 2.27180408239 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499825741491\n",
      "New best!\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 2.28658985297 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499799151078\n",
      "New best!\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 2.26140616735 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499765433292\n",
      "New best!\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 2.24945969979 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499732367523\n",
      "New best!\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 2.27394571702 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499668621305\n",
      "New best!\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 2.2695201834 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499622639992\n",
      "New best!\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 2.27130524715 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499600708317\n",
      "New best!\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 2.29323195219 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49951538876\n",
      "New best!\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 2.23846848408 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499474406722\n",
      "New best!\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 2.22787835201 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499446805447\n",
      "New best!\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 2.24317055146 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499346612308\n",
      "New best!\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 2.25318828424 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499342000838\n",
      "New best!\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 2.24786111911 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499207219857\n",
      "New best!\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 2.2332130154 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49914917709\n",
      "New best!\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 2.25301105181 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499097932372\n",
      "New best!\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 2.42770375013 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499047986222\n",
      "New best!\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 2.25767674843 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499191652954\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 2.25899410248 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499104670618\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 2.26830071608 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499009290092\n",
      "New best!\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 2.26146599849 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49890226824\n",
      "New best!\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 2.26537746588 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499328853664\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 2.2433493336 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498992374282\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 2.2372622331 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499064307803\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 2.2492805481 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499223565108\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 2.26628095309 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499407681645\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 2.26340945164 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499325869607\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 2.25875693162 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499407633111\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 2.25099146763 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499530215961\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 2.27615960042 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499747690114\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 2.25245666504 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499822139401\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 2.24652866522 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500208725054\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 2.26713806788 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500280048692\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 2.26079176664 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501045113263\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 2.28137333393 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501318848834\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 2.2504554987 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500845854428\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 2.2761018316 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501397872628\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 2.25855734746 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500843924449\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 2.30096338193 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501108047801\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 2.29196171761 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501258747066\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 2.25185723305 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501450231193\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 2.25220439831 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501416621372\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 2.25178320011 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501674191461\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 2.25230906804 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504085720981\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 2.26274349689 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502357652789\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 2.24752591848 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502132564273\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 2.24398961862 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50250285835\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 2.24959565004 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502793764394\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 2.23778134982 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503956105124\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 2.26515408357 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502421339887\n",
      "~Epoch 56~\n",
      "[--------- ] \n",
      "Finished. Took 2.24608588219 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502812309287\n",
      "~Epoch 57~\n",
      "[--------- ] \n",
      "Finished. Took 2.2459580342 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503207540282\n",
      "~Epoch 58~\n",
      "[--------- ] \n",
      "Finished. Took 2.24838664929 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503464162941\n",
      "~Epoch 59~\n",
      "[--------- ] \n",
      "Finished. Took 2.24606793722 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504243069594\n",
      "~Epoch 60~\n",
      "[--------- ] \n",
      "Finished. Took 2.24378839731 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503697899526\n",
      "~Epoch 61~\n",
      "[--------- ] \n",
      "Finished. Took 2.26489870151 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503614335276\n",
      "~Epoch 62~\n",
      "[--------- ] \n",
      "Finished. Took 2.25809711615 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50644068023\n",
      "~Epoch 63~\n",
      "[--------- ] \n",
      "Finished. Took 2.26744213502 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504134556273\n",
      "~Epoch 64~\n",
      "[--------- ] \n",
      "Finished. Took 2.25308461587 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508479219447\n",
      "~Epoch 65~\n",
      "[--------- ] \n",
      "Finished. Took 2.24941840172 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508034481054\n",
      "~Epoch 66~\n",
      "[--------- ] \n",
      "Finished. Took 2.24237541755 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508062246727\n",
      "Finished!\n",
      "Best error: 0.49890226824\n",
      "Checking...\n",
      "0 [-0.00992123] [-1]\n",
      "1 [ 0.02409328] [1]\n",
      "2 [-0.05632645] [-1]\n",
      "3 [ 0.00500576] [-1]\n",
      "4 [ 0.04234306] [-1]\n",
      "5 [ 0.0092659] [-1]\n",
      "6 [ 0.07561039] [1]\n",
      "Accuracy: train 0.539929947461 val 0.505633802817 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=20, model=nnb.SimpleRecurrence(insize=50, outsize=20, activation_func=nnb.activation.ReLU))[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=20, model=nnb.SimpleRecurrence(insize=50, outsize=20, activation_func=nnb.activation.ReLU))[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=20, model=nnb.SimpleRecurrence(insize=50, outsize=20, activation_func=nnb.activation.ReLU))[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*20, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model4 = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model4, s0, s1, s2, y, L2_reg=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 2.11676299969 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500007582211\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 2.08918610017 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500011072882\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 2.09595113198 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500030174807\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 2.10338778496 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50000375243\n",
      "New best!\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 2.11213076512 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500031647508\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 2.08398341735 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499997490661\n",
      "New best!\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 2.09848928452 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500025901847\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 2.09451918602 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499984544622\n",
      "New best!\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 2.11149510145 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499983630391\n",
      "New best!\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 2.11260908445 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49998343973\n",
      "New best!\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 2.0900385499 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499987441283\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 2.10354518493 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499993230567\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 2.08835528294 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499983305102\n",
      "New best!\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 2.11188966433 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499977581804\n",
      "New best!\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 2.0903072993 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499984655355\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 2.10814186732 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499982399065\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 2.09866069953 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499982023014\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 2.09362270037 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49999726566\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 2.09671121438 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49997153947\n",
      "New best!\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 2.0744386673 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499991533109\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 2.09614876509 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499965608608\n",
      "New best!\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 2.13153561751 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499971693883\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 2.14031606515 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499962152052\n",
      "New best!\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 2.15530863206 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499957295607\n",
      "New best!\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 2.1352534016 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499953805355\n",
      "New best!\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 2.08531423012 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499976673\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 2.09815765222 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499949372668\n",
      "New best!\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 2.08791468541 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499943554458\n",
      "New best!\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 2.11091153224 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499945543334\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 2.08199899991 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499944567949\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 2.30830775102 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499938837289\n",
      "New best!\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 2.08924280008 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499929506156\n",
      "New best!\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 2.10466845036 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499937911269\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 2.08261266549 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499989125211\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 2.11220731735 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499969277553\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 2.09562676748 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499916567892\n",
      "New best!\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 2.13010443052 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499917627869\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 2.13016538223 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499915305389\n",
      "New best!\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 2.11838653485 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499907109545\n",
      "New best!\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 2.14379841884 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499905121272\n",
      "New best!\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 2.1898140351 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49989240112\n",
      "New best!\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 2.15503805081 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499883043256\n",
      "New best!\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 2.14876478513 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499875403228\n",
      "New best!\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 2.09934798082 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499916006013\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 2.14625848134 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499859945272\n",
      "New best!\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 2.15407006343 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499850641634\n",
      "New best!\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 2.19478793144 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499866238587\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 2.1645833691 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499832412561\n",
      "New best!\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 2.16789328257 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499822570735\n",
      "New best!\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 2.1443323493 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49981652631\n",
      "New best!\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 2.11026828289 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499804385574\n",
      "New best!\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 2.0929960688 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499823071399\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 2.14298248291 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499857262882\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 2.08303624789 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499777253146\n",
      "New best!\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 2.1119502147 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499773894449\n",
      "New best!\n",
      "~Epoch 56~\n",
      "[--------- ] \n",
      "Finished. Took 2.14646735191 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499753530747\n",
      "New best!\n",
      "~Epoch 57~\n",
      "[--------- ] \n",
      "Finished. Took 2.12794890006 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499789634393\n",
      "~Epoch 58~\n",
      "[--------- ] \n",
      "Finished. Took 2.10633796851 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499734373202\n",
      "New best!\n",
      "~Epoch 59~\n",
      "[--------- ] \n",
      "Finished. Took 2.1060958306 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499713195231\n",
      "New best!\n",
      "~Epoch 60~\n",
      "[--------- ] \n",
      "Finished. Took 2.08885914882 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499721526559\n",
      "~Epoch 61~\n",
      "[--------- ] \n",
      "Finished. Took 2.12003193299 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49969057742\n",
      "New best!\n",
      "~Epoch 62~\n",
      "[--------- ] \n",
      "Finished. Took 2.08689090014 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499670497977\n",
      "New best!\n",
      "~Epoch 63~\n",
      "[--------- ] \n",
      "Finished. Took 2.13203473091 minutes.\n",
      "Evaluating...\n",
      "Error = 0.4996421112\n",
      "New best!\n",
      "~Epoch 64~\n",
      "[--------- ] \n",
      "Finished. Took 2.10370439688 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499671532344\n",
      "~Epoch 65~\n",
      "[--------- ] \n",
      "Finished. Took 2.11016976833 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499607586174\n",
      "New best!\n",
      "~Epoch 66~\n",
      "[--------- ] \n",
      "Finished. Took 2.09253563484 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499593232186\n",
      "New best!\n",
      "~Epoch 67~\n",
      "[--------- ] \n",
      "Finished. Took 2.09349698226 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499567840411\n",
      "New best!\n",
      "~Epoch 68~\n",
      "[--------- ] \n",
      "Finished. Took 2.09778743188 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499556199128\n",
      "New best!\n",
      "~Epoch 69~\n",
      "[--------- ] \n",
      "Finished. Took 2.09534246524 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499512054268\n",
      "New best!\n",
      "~Epoch 70~\n",
      "[--------- ] \n",
      "Finished. Took 2.08692908287 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499496805325\n",
      "New best!\n",
      "~Epoch 71~\n",
      "[--------- ] \n",
      "Finished. Took 2.10822341442 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499465321231\n",
      "New best!\n",
      "~Epoch 72~\n",
      "[--------- ] \n",
      "Finished. Took 2.10902835131 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499439827427\n",
      "New best!\n",
      "~Epoch 73~\n",
      "[--------- ] \n",
      "Finished. Took 2.12338110209 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499396223895\n",
      "New best!\n",
      "~Epoch 74~\n",
      "[--------- ] \n",
      "Finished. Took 2.07978994846 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499368614742\n",
      "New best!\n",
      "~Epoch 75~\n",
      "[--------- ] \n",
      "Finished. Took 2.14894528389 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499333734988\n",
      "New best!\n",
      "~Epoch 76~\n",
      "[--------- ] \n",
      "Finished. Took 2.16978758176 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499306676958\n",
      "New best!\n",
      "~Epoch 77~\n",
      "[--------- ] \n",
      "Finished. Took 2.17855916421 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499268013515\n",
      "New best!\n",
      "~Epoch 78~\n",
      "[--------- ] \n",
      "Finished. Took 2.18162244956 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499276733597\n",
      "~Epoch 79~\n",
      "[--------- ] \n",
      "Finished. Took 2.20525831381 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49921133269\n",
      "New best!\n",
      "~Epoch 80~\n",
      "[--------- ] \n",
      "Finished. Took 2.70182991823 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499165878727\n",
      "New best!\n",
      "~Epoch 81~\n",
      "[--------- ] \n",
      "Finished. Took 3.0655498306 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499124504446\n",
      "New best!\n",
      "~Epoch 82~\n",
      "[--------- ] \n",
      "Finished. Took 2.47923826774 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499076045151\n",
      "New best!\n",
      "~Epoch 83~\n",
      "[--------- ] \n",
      "Finished. Took 2.87967254718 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499179210183\n",
      "~Epoch 84~\n",
      "[--------- ] \n",
      "Finished. Took 2.91597326994 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499014576497\n",
      "New best!\n",
      "~Epoch 85~\n",
      "[--------- ] \n",
      "Finished. Took 3.22293498119 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498958555304\n",
      "New best!\n",
      "~Epoch 86~\n",
      "[--------- ] \n",
      "Finished. Took 2.96152823369 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498912211168\n",
      "New best!\n",
      "~Epoch 87~\n",
      "[--------- ] \n",
      "Finished. Took 3.0920374155 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498865114358\n",
      "New best!\n",
      "~Epoch 88~\n",
      "[--------- ] \n",
      "Finished. Took 2.94777288437 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49882125141\n",
      "New best!\n",
      "~Epoch 89~\n",
      "[--------- ] \n",
      "Finished. Took 3.23306883176 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498788995549\n",
      "New best!\n",
      "~Epoch 90~\n",
      "[--------- ] \n",
      "Finished. Took 3.06091201703 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498784361995\n",
      "New best!\n",
      "~Epoch 91~\n",
      "[--------- ] \n",
      "Finished. Took 3.24701723258 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49871067057\n",
      "New best!\n",
      "~Epoch 92~\n",
      "[--------- ] \n",
      "Finished. Took 3.29998898109 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498661650044\n",
      "New best!\n",
      "~Epoch 93~\n",
      "[--------- ] \n",
      "Finished. Took 3.23369065126 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498633209053\n",
      "New best!\n",
      "~Epoch 94~\n",
      "[--------- ] \n",
      "Finished. Took 3.05871660312 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498570683939\n",
      "New best!\n",
      "~Epoch 95~\n",
      "[--------- ] \n",
      "Finished. Took 3.23506915172 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498578297763\n",
      "~Epoch 96~\n",
      "[--------- ] \n",
      "Finished. Took 3.19297596614 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498526964413\n",
      "New best!\n",
      "~Epoch 97~\n",
      "[--------- ] \n",
      "Finished. Took 3.12613131603 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498553467694\n",
      "~Epoch 98~\n",
      "[--------- ] \n",
      "Finished. Took 2.96797825098 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498584326403\n",
      "~Epoch 99~\n",
      "[--------- ] \n",
      "Finished. Took 3.32691413164 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498461249745\n",
      "New best!\n",
      "~Epoch 100~\n",
      "[--------- ] \n",
      "Finished. Took 3.10469580094 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498944605038\n",
      "~Epoch 101~\n",
      "[--------- ] \n",
      "Finished. Took 3.00155956745 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498397499767\n",
      "New best!\n",
      "~Epoch 102~\n",
      "[--------- ] \n",
      "Finished. Took 2.96833131711 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498624072054\n",
      "~Epoch 103~\n",
      "[--------- ] \n",
      "Finished. Took 2.11900810003 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498516470992\n",
      "~Epoch 104~\n",
      "[--------- ] \n",
      "Finished. Took 2.13914348284 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49842210448\n",
      "~Epoch 105~\n",
      "[--------- ] \n",
      "Finished. Took 3.01517489751 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498817471283\n",
      "~Epoch 106~\n",
      "[--------- ] \n",
      "Finished. Took 3.4263517499 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49850065489\n",
      "~Epoch 107~\n",
      "[--------- ] \n",
      "Finished. Took 3.21878430049 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498635413809\n",
      "~Epoch 108~\n",
      "[--------- ] \n",
      "Finished. Took 3.06113028526 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498814943782\n",
      "~Epoch 109~\n",
      "[--------- ] \n",
      "Finished. Took 3.34404060046 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498701165879\n",
      "~Epoch 110~\n",
      "[--------- ] \n",
      "Finished. Took 3.32384693623 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49863744021\n",
      "~Epoch 111~\n",
      "[--------- ] \n",
      "Finished. Took 3.10865125259 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498693401589\n",
      "~Epoch 112~\n",
      "[--------- ] \n",
      "Finished. Took 3.17475560109 minutes.\n",
      "Evaluating...\n",
      "Error = 0.498990378101\n",
      "~Epoch 113~\n",
      "[--------- ] \n",
      "Finished. Took 2.99081803163 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499187989651\n",
      "~Epoch 114~\n",
      "[--------- ] \n",
      "Finished. Took 3.08220423063 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499369488526\n",
      "~Epoch 115~\n",
      "[--------- ] \n",
      "Finished. Took 3.09623705149 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49904731954\n",
      "~Epoch 116~\n",
      "[--------- ] \n",
      "Finished. Took 3.06639763117 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499117980634\n",
      "~Epoch 117~\n",
      "[--------- ] \n",
      "Finished. Took 3.19093244871 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499217830877\n",
      "~Epoch 118~\n",
      "[--------- ] \n",
      "Finished. Took 3.26605400244 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500984987627\n",
      "~Epoch 119~\n",
      "[--------- ] \n",
      "Finished. Took 2.99420148532 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499428937006\n",
      "~Epoch 120~\n",
      "[--------- ] \n",
      "Finished. Took 3.29581988255 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500219702093\n",
      "~Epoch 121~\n",
      "[--------- ] \n",
      "Finished. Took 3.76728701989 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499677904505\n",
      "~Epoch 122~\n",
      "[--------- ] \n",
      "Finished. Took 3.23100563288 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499940163245\n",
      "~Epoch 123~\n",
      "[--------- ] \n",
      "Finished. Took 3.10756948392 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499994698822\n",
      "~Epoch 124~\n",
      "[--------- ] \n",
      "Finished. Took 2.86480760177 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500779088371\n",
      "~Epoch 125~\n",
      "[--------- ] \n",
      "Finished. Took 3.08618880113 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500932792235\n",
      "~Epoch 126~\n",
      "[--------- ] \n",
      "Finished. Took 3.09277986685 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50152770838\n",
      "~Epoch 127~\n",
      "[--------- ] \n",
      "Finished. Took 3.32299925089 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500505930051\n",
      "~Epoch 128~\n",
      "[--------- ] \n",
      "Finished. Took 2.94110781352 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500623083218\n",
      "~Epoch 129~\n",
      "[--------- ] \n",
      "Finished. Took 3.00878461599 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50068404345\n",
      "~Epoch 130~\n",
      "[--------- ] \n",
      "Finished. Took 2.81355164846 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500847138071\n",
      "~Epoch 131~\n",
      "[--------- ] \n",
      "Finished. Took 3.10717696746 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501235209086\n",
      "~Epoch 132~\n",
      "[--------- ] \n",
      "Finished. Took 2.97429149946 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503393227813\n",
      "~Epoch 133~\n",
      "[--------- ] \n",
      "Finished. Took 2.86658405066 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50129263867\n",
      "~Epoch 134~\n",
      "[--------- ] \n",
      "Finished. Took 2.24099076589 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501256859156\n",
      "~Epoch 135~\n",
      "[--------- ] \n",
      "Finished. Took 2.94436370134 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501765609567\n",
      "~Epoch 136~\n",
      "[--------- ] \n",
      "Finished. Took 3.1037912329 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501893216576\n",
      "~Epoch 137~\n",
      "[--------- ] \n",
      "Finished. Took 3.18777986765 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501960833255\n",
      "~Epoch 138~\n",
      "[--------- ] \n",
      "Finished. Took 2.97692321936 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501934281686\n",
      "~Epoch 139~\n",
      "[--------- ] \n",
      "Finished. Took 3.09279268583 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502134125772\n",
      "~Epoch 140~\n",
      "[--------- ] \n",
      "Finished. Took 2.81886444887 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502468634784\n",
      "~Epoch 141~\n",
      "[--------- ] \n",
      "Finished. Took 2.9575966994 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503598935212\n",
      "Finished!\n",
      "Best error: 0.498397499767\n",
      "Checking...\n",
      "0 [-0.00815107] [-1]\n",
      "1 [ 0.03148589] [1]\n",
      "2 [-0.06466909] [-1]\n",
      "3 [-0.0115813] [-1]\n",
      "4 [ 0.02281854] [-1]\n",
      "5 [-0.05501135] [-1]\n",
      "6 [ 0.08905659] [1]\n",
      "Accuracy: train 0.554640980736 val 0.516197183099 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C1 = nnb.PerceptronLayer(insize=3*10, outsize=10, activation_func=nnb.activation.ReLU)\n",
    "C2 = nnb.PerceptronLayer(insize=10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model5 = (L0 & L1 & L2) | CC | C1 | C2\n",
    "try_model(model5, s0, s1, s2, y, L2_reg=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 1.55834203561 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500311841604\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 1.54683081706 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499995299959\n",
      "New best!\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 1.53445533514 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502030729056\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 1.58709093332 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500193613734\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 1.54217398167 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500135160128\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 1.59590096474 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500135804885\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 1.53472296794 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50370177462\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 1.59927145243 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499999532169\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 1.55461763541 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500021580757\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 1.59276033243 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499953049277\n",
      "New best!\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 1.57640031576 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499947806649\n",
      "New best!\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 1.58895499706 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500778129906\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 1.61597255071 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499939704566\n",
      "New best!\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 1.59670550028 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500227831582\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 1.60661309958 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500030075973\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 1.56716481845 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500710762883\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 1.60737083356 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500465737519\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 1.58723688523 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501356442353\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 1.62112956444 minutes.\n",
      "Evaluating...\n",
      "Error = 0.4999198067\n",
      "New best!\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 1.61424878438 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500041370619\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 1.60543150107 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499893750495\n",
      "New best!\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 1.60228501956 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500093448871\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 1.58882656495 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500489973828\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 1.58597764969 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499878257367\n",
      "New best!\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 1.60231416623 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500195584093\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 1.55827680031 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49989128385\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 1.60392450094 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499898486376\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 1.55532705386 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499959921631\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 1.59309613307 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500381172275\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 1.54286791881 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500071703477\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 1.59357093573 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501273071081\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 1.55595022043 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500138875705\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 1.59574273427 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500583958451\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 1.55574526787 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499806271478\n",
      "New best!\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 1.58610779842 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499975222963\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 1.561500748 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500093650038\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 1.5771290501 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499898816378\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 1.55372663339 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500128780408\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 1.58847483397 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499835259484\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 1.54840625127 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499819838636\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 1.58402149677 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499756766317\n",
      "New best!\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 1.5598617355 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500217588801\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 1.57465941509 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499774717386\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 1.56037985086 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499742530742\n",
      "New best!\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 1.56337261995 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499773416364\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 1.58399676879 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499761881795\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 1.57018913428 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499727384334\n",
      "New best!\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 1.5408847332 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49977714371\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 1.56977270047 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499738445151\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 1.53834146659 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499834568573\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 1.58212561607 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499716407106\n",
      "New best!\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 1.55177883307 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499979491556\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 1.59738038381 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499950350915\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 1.55573216677 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500828248948\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 1.56707708438 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499790269654\n",
      "~Epoch 56~\n",
      "[--------- ] \n",
      "Finished. Took 1.55306899945 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499919894118\n",
      "~Epoch 57~\n",
      "[--------- ] \n",
      "Finished. Took 1.5667468826 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499793843223\n",
      "~Epoch 58~\n",
      "[--------- ] \n",
      "Finished. Took 1.54885170062 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499804016974\n",
      "~Epoch 59~\n",
      "[--------- ] \n",
      "Finished. Took 1.58368796905 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499740159899\n",
      "~Epoch 60~\n",
      "[--------- ] \n",
      "Finished. Took 1.54950601657 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500219592686\n",
      "~Epoch 61~\n",
      "[--------- ] \n",
      "Finished. Took 1.57651950121 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499778993742\n",
      "~Epoch 62~\n",
      "[--------- ] \n",
      "Finished. Took 1.57540063063 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499722511046\n",
      "~Epoch 63~\n",
      "[--------- ] \n",
      "Finished. Took 1.58660744826 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500780804331\n",
      "~Epoch 64~\n",
      "[--------- ] \n",
      "Finished. Took 1.76646243334 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499780564398\n",
      "~Epoch 65~\n",
      "[--------- ] \n",
      "Finished. Took 1.59265251557 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500554501476\n",
      "~Epoch 66~\n",
      "[--------- ] \n",
      "Finished. Took 1.56543981632 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501430164916\n",
      "~Epoch 67~\n",
      "[--------- ] \n",
      "Finished. Took 1.58987116814 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499821878894\n",
      "~Epoch 68~\n",
      "[--------- ] \n",
      "Finished. Took 1.57729158401 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49976128908\n",
      "~Epoch 69~\n",
      "[--------- ] \n",
      "Finished. Took 1.58362786373 minutes.\n",
      "Evaluating...\n",
      "Error = 0.5006137083\n",
      "~Epoch 70~\n",
      "[--------- ] \n",
      "Finished. Took 1.56591366927 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499770209336\n",
      "~Epoch 71~\n",
      "[--------- ] \n",
      "Finished. Took 1.58976693551 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499791456914\n",
      "~Epoch 72~\n",
      "[--------- ] \n",
      "Finished. Took 1.56011581421 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499822769176\n",
      "~Epoch 73~\n",
      "[--------- ] \n",
      "Finished. Took 1.58068164984 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501838974899\n",
      "~Epoch 74~\n",
      "[--------- ] \n",
      "Finished. Took 1.58317799966 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499965972429\n",
      "~Epoch 75~\n",
      "[--------- ] \n",
      "Finished. Took 1.58472603559 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500102767063\n",
      "~Epoch 76~\n",
      "[--------- ] \n",
      "Finished. Took 1.57937526703 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500316609413\n",
      "~Epoch 77~\n",
      "[--------- ] \n",
      "Finished. Took 1.56370068391 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500128080183\n",
      "~Epoch 78~\n",
      "[--------- ] \n",
      "Finished. Took 1.56565039953 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499896957482\n",
      "~Epoch 79~\n",
      "[--------- ] \n",
      "Finished. Took 1.5721527338 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499875288468\n",
      "~Epoch 80~\n",
      "[--------- ] \n",
      "Finished. Took 1.57977128029 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500376874341\n",
      "~Epoch 81~\n",
      "[--------- ] \n",
      "Finished. Took 1.59363644918 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50001086273\n",
      "~Epoch 82~\n",
      "[--------- ] \n",
      "Finished. Took 1.59090679884 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500219907173\n",
      "~Epoch 83~\n",
      "[--------- ] \n",
      "Finished. Took 1.57369099855 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499894792297\n",
      "~Epoch 84~\n",
      "[--------- ] \n",
      "Finished. Took 1.56661738157 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499943093821\n",
      "~Epoch 85~\n",
      "[--------- ] \n",
      "Finished. Took 1.57371700207 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499918567848\n",
      "~Epoch 86~\n",
      "[--------- ] \n",
      "Finished. Took 1.57625745138 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500215437153\n",
      "~Epoch 87~\n",
      "[--------- ] \n",
      "Finished. Took 1.5851273338 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500237624004\n",
      "~Epoch 88~\n",
      "[--------- ] \n",
      "Finished. Took 1.58223753373 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500199648759\n",
      "~Epoch 89~\n",
      "[--------- ] \n",
      "Finished. Took 1.57917696635 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500342060412\n",
      "~Epoch 90~\n",
      "[--------- ] \n",
      "Finished. Took 1.58321676652 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500026306752\n",
      "~Epoch 91~\n",
      "[--------- ] \n",
      "Finished. Took 1.67852081458 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500101374018\n",
      "Finished!\n",
      "Best error: 0.499716407106\n",
      "Checking...\n",
      "0 [ 0.00454902] [-1]\n",
      "1 [-0.00159105] [1]\n",
      "2 [ 0.00317867] [-1]\n",
      "3 [ 0.00290131] [-1]\n",
      "4 [ 0.00325541] [-1]\n",
      "5 [ 0.00294062] [-1]\n",
      "6 [ 0.00335912] [1]\n",
      "Accuracy: train 0.521716287215 val 0.497183098592 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10)[-1]\n",
    "\n",
    "C1 = nnb.PerceptronLayer(insize=2*10, outsize=10)  #, activation_func=nnb.activation.ReLU)\n",
    "qamodel = (L0 & L1) | nnb.ConcatenationModel() | C1\n",
    "\n",
    "C2 = nnb.PerceptronLayer(insize=2*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model3 = (qamodel & L2) | nnb.ConcatenationModel() | C2\n",
    "try_model(model3, s0, s1, s2, y, L2_reg=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n",
      "[--------- ] \n",
      "Finished. Took 2.25736728112 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500083714276\n",
      "New best!\n",
      "~Epoch 2~\n",
      "[--------- ] \n",
      "Finished. Took 2.20320586761 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500081086591\n",
      "New best!\n",
      "~Epoch 3~\n",
      "[--------- ] \n",
      "Finished. Took 2.61474933227 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50009305823\n",
      "~Epoch 4~\n",
      "[--------- ] \n",
      "Finished. Took 2.33273438613 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500091675891\n",
      "~Epoch 5~\n",
      "[--------- ] \n",
      "Finished. Took 2.44618324836 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500076664674\n",
      "New best!\n",
      "~Epoch 6~\n",
      "[--------- ] \n",
      "Finished. Took 2.35433656772 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500079727774\n",
      "~Epoch 7~\n",
      "[--------- ] \n",
      "Finished. Took 2.37073853413 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500073095526\n",
      "New best!\n",
      "~Epoch 8~\n",
      "[--------- ] \n",
      "Finished. Took 2.47728745143 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500111267567\n",
      "~Epoch 9~\n",
      "[--------- ] \n",
      "Finished. Took 2.66203231812 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500032738551\n",
      "New best!\n",
      "~Epoch 10~\n",
      "[--------- ] \n",
      "Finished. Took 2.5184095184 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500023613986\n",
      "New best!\n",
      "~Epoch 11~\n",
      "[--------- ] \n",
      "Finished. Took 4.14127526681 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50000120968\n",
      "New best!\n",
      "~Epoch 12~\n",
      "[--------- ] \n",
      "Finished. Took 2.52550470034 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500002622471\n",
      "~Epoch 13~\n",
      "[--------- ] \n",
      "Finished. Took 2.67198703289 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499958107676\n",
      "New best!\n",
      "~Epoch 14~\n",
      "[--------- ] \n",
      "Finished. Took 2.74413658381 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499936681747\n",
      "New best!\n",
      "~Epoch 15~\n",
      "[--------- ] \n",
      "Finished. Took 2.81313434839 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499901946981\n",
      "New best!\n",
      "~Epoch 16~\n",
      "[--------- ] \n",
      "Finished. Took 2.67671768268 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499897252024\n",
      "New best!\n",
      "~Epoch 17~\n",
      "[--------- ] \n",
      "Finished. Took 2.62144664923 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499860417806\n",
      "New best!\n",
      "~Epoch 18~\n",
      "[--------- ] \n",
      "Finished. Took 2.57571214835 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499842130654\n",
      "New best!\n",
      "~Epoch 19~\n",
      "[--------- ] \n",
      "Finished. Took 2.90705345074 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499838925428\n",
      "New best!\n",
      "~Epoch 20~\n",
      "[--------- ] \n",
      "Finished. Took 2.30370981693 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499845050802\n",
      "~Epoch 21~\n",
      "[--------- ] \n",
      "Finished. Took 2.51796194712 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499778796457\n",
      "New best!\n",
      "~Epoch 22~\n",
      "[--------- ] \n",
      "Finished. Took 3.23842129707 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499772513608\n",
      "New best!\n",
      "~Epoch 23~\n",
      "[--------- ] \n",
      "Finished. Took 3.12773536841 minutes.\n",
      "Evaluating...\n",
      "Error = 0.4998453114\n",
      "~Epoch 24~\n",
      "[--------- ] \n",
      "Finished. Took 3.31404003302 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499752632384\n",
      "New best!\n",
      "~Epoch 25~\n",
      "[--------- ] \n",
      "Finished. Took 3.33617029985 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499750475177\n",
      "New best!\n",
      "~Epoch 26~\n",
      "[--------- ] \n",
      "Finished. Took 3.13705224991 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500087978367\n",
      "~Epoch 27~\n",
      "[--------- ] \n",
      "Finished. Took 3.12288286686 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499873382619\n",
      "~Epoch 28~\n",
      "[--------- ] \n",
      "Finished. Took 3.34401575327 minutes.\n",
      "Evaluating...\n",
      "Error = 0.49985212135\n",
      "~Epoch 29~\n",
      "[--------- ] \n",
      "Finished. Took 3.2355145812 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499898631061\n",
      "~Epoch 30~\n",
      "[--------- ] \n",
      "Finished. Took 3.11980156501 minutes.\n",
      "Evaluating...\n",
      "Error = 0.499919716715\n",
      "~Epoch 31~\n",
      "[--------- ] \n",
      "Finished. Took 3.22553378344 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500108431683\n",
      "~Epoch 32~\n",
      "[--------- ] \n",
      "Finished. Took 3.11567056576 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500173625301\n",
      "~Epoch 33~\n",
      "[--------- ] \n",
      "Finished. Took 3.34783183336 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500298212851\n",
      "~Epoch 34~\n",
      "[--------- ] \n",
      "Finished. Took 3.21156521638 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500317659938\n",
      "~Epoch 35~\n",
      "[--------- ] \n",
      "Finished. Took 3.07297173341 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500812579884\n",
      "~Epoch 36~\n",
      "[--------- ] \n",
      "Finished. Took 3.32974489927 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500539165705\n",
      "~Epoch 37~\n",
      "[--------- ] \n",
      "Finished. Took 3.15928423405 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500765486903\n",
      "~Epoch 38~\n",
      "[--------- ] \n",
      "Finished. Took 2.08131418228 minutes.\n",
      "Evaluating...\n",
      "Error = 0.500963144486\n",
      "~Epoch 39~\n",
      "[--------- ] \n",
      "Finished. Took 2.09305893183 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501466994263\n",
      "~Epoch 40~\n",
      "[--------- ] \n",
      "Finished. Took 2.11660449902 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501630788696\n",
      "~Epoch 41~\n",
      "[--------- ] \n",
      "Finished. Took 2.36993528207 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501516149128\n",
      "~Epoch 42~\n",
      "[--------- ] \n",
      "Finished. Took 3.41155430079 minutes.\n",
      "Evaluating...\n",
      "Error = 0.501913587645\n",
      "~Epoch 43~\n",
      "[--------- ] \n",
      "Finished. Took 3.18511890173 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502506958305\n",
      "~Epoch 44~\n",
      "[--------- ] \n",
      "Finished. Took 2.49764923652 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502246989136\n",
      "~Epoch 45~\n",
      "[--------- ] \n",
      "Finished. Took 2.11711676915 minutes.\n",
      "Evaluating...\n",
      "Error = 0.502602851546\n",
      "~Epoch 46~\n",
      "[--------- ] \n",
      "Finished. Took 2.07267919779 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50303508568\n",
      "~Epoch 47~\n",
      "[--------- ] \n",
      "Finished. Took 2.1161740462 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503095908598\n",
      "~Epoch 48~\n",
      "[--------- ] \n",
      "Finished. Took 2.07570571502 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503421017881\n",
      "~Epoch 49~\n",
      "[--------- ] \n",
      "Finished. Took 2.07219626506 minutes.\n",
      "Evaluating...\n",
      "Error = 0.503691677979\n",
      "~Epoch 50~\n",
      "[--------- ] \n",
      "Finished. Took 2.07420936823 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504012745602\n",
      "~Epoch 51~\n",
      "[--------- ] \n",
      "Finished. Took 2.27226396799 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504206489885\n",
      "~Epoch 52~\n",
      "[--------- ] \n",
      "Finished. Took 2.10213343302 minutes.\n",
      "Evaluating...\n",
      "Error = 0.504677581757\n",
      "~Epoch 53~\n",
      "[--------- ] \n",
      "Finished. Took 2.1010071675 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50539388479\n",
      "~Epoch 54~\n",
      "[--------- ] \n",
      "Finished. Took 2.10576785008 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505327514181\n",
      "~Epoch 55~\n",
      "[--------- ] \n",
      "Finished. Took 2.17885338465 minutes.\n",
      "Evaluating...\n",
      "Error = 0.505773204476\n",
      "~Epoch 56~\n",
      "[--------- ] \n",
      "Finished. Took 2.12705081701 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506065245363\n",
      "~Epoch 57~\n",
      "[--------- ] \n",
      "Finished. Took 2.18036298354 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509501219699\n",
      "~Epoch 58~\n",
      "[--------- ] \n",
      "Finished. Took 2.08853040139 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506572791008\n",
      "~Epoch 59~\n",
      "[--------- ] \n",
      "Finished. Took 2.08637793461 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50738068987\n",
      "~Epoch 60~\n",
      "[--------- ] \n",
      "Finished. Took 2.07309718529 minutes.\n",
      "Evaluating...\n",
      "Error = 0.506986624246\n",
      "~Epoch 61~\n",
      "[--------- ] \n",
      "Finished. Took 2.20610108376 minutes.\n",
      "Evaluating...\n",
      "Error = 0.509942387662\n",
      "~Epoch 62~\n",
      "[--------- ] \n",
      "Finished. Took 2.24246125221 minutes.\n",
      "Evaluating...\n",
      "Error = 0.50708947814\n",
      "~Epoch 63~\n",
      "[--------- ] \n",
      "Finished. Took 2.26481823126 minutes.\n",
      "Evaluating...\n",
      "Error = 0.507300805725\n",
      "~Epoch 64~\n",
      "[--------- ] \n",
      "Finished. Took 2.34024781386 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508718328823\n",
      "~Epoch 65~\n",
      "[--------- ] \n",
      "Finished. Took 2.1144669493 minutes.\n",
      "Evaluating...\n",
      "Error = 0.508244507692\n",
      "Finished!\n",
      "Best error: 0.499750475177\n",
      "Checking...\n",
      "0 [ 0.00606586] [-1]\n",
      "1 [ 0.0079491] [1]\n",
      "2 [  9.13063618e-05] [-1]\n",
      "3 [ 0.01027014] [-1]\n",
      "4 [ 0.02585167] [-1]\n",
      "5 [ 0.02105067] [-1]\n",
      "6 [ 0.08665964] [1]\n",
      "Accuracy: train 0.543607705779 val 0.514084507042 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "L0 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L1 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "L2 = nnb.InputLayer(ndim=2) | nnb.RecurrentNeuralNetwork(insize=50, outsize=10, model=nnb.SimpleRecurrence(insize=50, outsize=10, activation_func=nnb.activation.ReLU))[-1]\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model6 = (L0 & L1 & L2) | CC | C2\n",
    "try_model(model6, s0, s1, s2, y, L2_reg=1e-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN partial train, full val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def check_model(model, s0, s1, s2, y, val_on_train=False):\n",
    "    y = y*2-1  # classify as -1/+1 for tanh activation    \n",
    "    \n",
    "    n = s0.shape[0]\n",
    "    tdata = [[s0[i], s1[i], s2[i], np.array([y[i]], dtype='int32')] for i in range(n)]\n",
    "    n_train = int(n*0.8) if not val_on_train else 0    \n",
    "    if n_train > 0:\n",
    "        tdata_train = balance_dataset(tdata[:n_train], y[:n_train])\n",
    "        tdata_val = balance_dataset(tdata[n_train:], y[n_train:])\n",
    "    else:\n",
    "        n_train = n\n",
    "        tdata_train = balance_dataset(tdata, y)\n",
    "        tdata_val = balance_dataset(tdata, y)\n",
    "    \n",
    "    ff = model.compile()\n",
    "    def eval_dataset(ff, tdata):\n",
    "        return np.array([ff(tdata[i][0], tdata[i][1], tdata[i][2])[0] for i in range(len(tdata))])\n",
    "    def accuracy(tdata, yy):\n",
    "        y = np.array([t[-1][0] for t in tdata])\n",
    "        n_cor = np.sum(y[yy > 0]+1)/2 + np.sum(-y[yy < 0]+1)/2\n",
    "        return n_cor / float(np.shape(yy)[0])\n",
    "    print('Accuracy:',\n",
    "          'train', accuracy(tdata_train, eval_dataset(ff, tdata_train)),\n",
    "          'val', accuracy(tdata_val, eval_dataset(ff, tdata_val)),\n",
    "          'val_base', accuracy(tdata_val, np.zeros(np.shape(tdata_val)[0])-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: train 0.555077452668 val 0.551369863014 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "check_model(model2, s0[500:1500], s1[500:1500], s2[500:1500], y[500:1500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: train 0.496322241681 val 0.519718309859 val_base 0.5\n"
     ]
    }
   ],
   "source": [
    "check_model(model_b200, s0, s1, s2, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing...\n",
      "Cost function...\n",
      "Compiling...\n",
      "Training...\n",
      "~Epoch 1~\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Output dimensions are not valid 1x-1\nApply node that caused the error: ConvOp{('imshp', (1, 50, None)),('kshp', (50, 3)),('nkern', 10),('bsize', 1),('dx', 1),('dy', 1),('out_mode', 'valid'),('unroll_batch', None),('unroll_kern', None),('unroll_patch', True),('imshp_logical', (1, 50, None)),('kshp_logical', (50, 3)),('kshp_logical_top_aligned', True)}(InplaceDimShuffle{x,x,1,0}.0, InplaceDimShuffle{0,x,1,2}.0)\nInputs types: [TensorType(float64, (True, True, False, False)), TensorType(float64, (False, True, False, False))]\nInputs shapes: [(1, 1, 50, 1), (10, 1, 50, 3)]\nInputs strides: [(400, 400, 8, 8), (1200, 1200, 24, 8)]\nInputs values: ['not shown', 'not shown']\n\nBacktrace when the node is created:\n  File \"build/bdist.linux-x86_64/egg/nnb/nn_model.py\", line 995, in apply\n    subsample=(1, stride)\n\nDebugprint of the apply node: \nConvOp{('imshp', (1, 50, None)),('kshp', (50, 3)),('nkern', 10),('bsize', 1),('dx', 1),('dy', 1),('out_mode', 'valid'),('unroll_batch', None),('unroll_kern', None),('unroll_patch', True),('imshp_logical', (1, 50, None)),('kshp_logical', (50, 3)),('kshp_logical_top_aligned', True)} [@A] <TensorType(float64, (True, False, True, False))> ''   \n |InplaceDimShuffle{x,x,1,0} [@B] <TensorType(float64, (True, True, False, False))> ''   \n | |input [@C] <TensorType(float64, matrix)>\n |InplaceDimShuffle{0,x,1,2} [@D] <TensorType(float64, (False, True, False, False))> ''   \n   |W [@E] <TensorType(float64, 3D)>\n\nStorage map footprint:\n - TensorConstant{-3}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - b, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - TensorConstant{1.0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - W, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - TensorConstant{(1, 1, 1) of 0.5}, Shape: (1, 1, 1), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - TensorConstant{-inf}, Shape: (1,), ElemSize: 4 Byte(s), TotalSize: 4.0 Byte(s)\n - <TensorType(float64, 3D)>, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - <TensorType(float64, 3D)>, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - b, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - b, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - Constant{-1}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - input, Shape: (461, 50), ElemSize: 8 Byte(s), TotalSize: 184400 Byte(s)\n - input, Shape: (1,), ElemSize: 4 Byte(s), TotalSize: 4 Byte(s)\n - b, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - Constant{0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - TensorConstant{0}, Shape: (1,), ElemSize: 1 Byte(s), TotalSize: 1.0 Byte(s)\n - TensorConstant{(1, 1, 1, 1) of 0.5}, Shape: (1, 1, 1, 1), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - MakeVector.0, Shape: (2,), ElemSize: 8 Byte(s), TotalSize: 16 Byte(s)\n - W, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - TensorConstant{(1,) of 2.0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - input, Shape: (1, 50), ElemSize: 8 Byte(s), TotalSize: 400 Byte(s)\n - W, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - input, Shape: (11, 50), ElemSize: 8 Byte(s), TotalSize: 4400 Byte(s)\n - TensorConstant{1}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - TensorConstant{(1,) of 0.5}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - max, Shape: (10, 10), ElemSize: 8 Byte(s), TotalSize: 800 Byte(s)\n - InplaceDimShuffle{x,1,x,3}.0, Shape: (1, 10, 1, 9), ElemSize: 8 Byte(s), TotalSize: 720 Byte(s)\n - Shape_i{2}.0, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - MakeVector.0, Shape: (2,), ElemSize: 8 Byte(s), TotalSize: 16 Byte(s)\n - Elemwise{Composite{(i0 + ((i1 + i2) // i0))}}[(0, 2)].0, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - Alloc.0, Shape: (10, 10), ElemSize: 8 Byte(s), TotalSize: 800 Byte(s)\n - TensorConstant{10}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - TensorConstant{0.0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - <TensorType(float64, vector)>, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - <TensorType(float64, vector)>, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - <TensorType(float64, matrix)>, Shape: (30, 1), ElemSize: 8 Byte(s), TotalSize: 240 Byte(s)\n - <TensorType(float64, vector)>, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - Reshape{3}.0, Shape: (10, 1, 10), ElemSize: 8 Byte(s), TotalSize: 800 Byte(s)\n - Elemwise{Composite{(i0 / Cast{float64}(i1))}}.0, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - <TensorType(float64, 3D)>, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - Subtensor{int64}.0, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - W, Shape: (30, 1), ElemSize: 8 Byte(s), TotalSize: 240 Byte(s)\n - TensorConstant{(1, 1) of 0.0}, Shape: (1, 1), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - Alloc.0, Shape: (30,), ElemSize: 8 Byte(s), TotalSize: 240 Byte(s)\n - W.T, Shape: (1, 30), ElemSize: 8 Byte(s), TotalSize: 240 Byte(s)\n - TensorConstant{(1,) of 1.0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - <TensorType(float64, vector)>, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - InplaceDimShuffle{x,x,1,0}.0, Shape: (1, 1, 50, 11), ElemSize: 8 Byte(s), TotalSize: 4400 Byte(s)\n - InplaceDimShuffle{0,x,1,2}.0, Shape: (10, 1, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - InplaceDimShuffle{x,x,1,0}.0, Shape: (1, 1, 50, 1), ElemSize: 8 Byte(s), TotalSize: 400 Byte(s)\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-be51c7ae7665>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mL\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m&\u001b[0m \u001b[0mL\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m&\u001b[0m \u001b[0mL\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m|\u001b[0m \u001b[0mCC\u001b[0m \u001b[1;33m|\u001b[0m \u001b[0mC2\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0mtry_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms0\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m200\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms1\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m200\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms2\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m200\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m200\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-7-cb429b599bcb>\u001b[0m in \u001b[0;36mtry_model\u001b[1;34m(model, s0, s1, s2, y, val_on_train, lrate, L2_reg)\u001b[0m\n\u001b[0;32m     34\u001b[0m                                           \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mn_train\u001b[0m\u001b[1;33m//\u001b[0m\u001b[1;36m100\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mn_train\u001b[0m \u001b[1;33m>=\u001b[0m \u001b[1;36m100\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mn_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m                                           epochs_num=200, max_no_improve=40)\n\u001b[1;32m---> 36\u001b[1;33m     \u001b[0mtrain_sup\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m     \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Checking...'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/NNBlocks-0.1.a.dev-py2.7.egg/nnb/train/train_supervisor.pyc\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    174\u001b[0m                 \u001b[0msi\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mi\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    175\u001b[0m                 \u001b[0mei\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 176\u001b[1;33m                 \u001b[0mtrainer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msi\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mei\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    177\u001b[0m                 \u001b[0mfracs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0miterations\u001b[0m \u001b[1;33m/\u001b[0m \u001b[1;36m10\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    178\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mfracs\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mi\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mfracs\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/NNBlocks-0.1.a.dev-py2.7.egg/nnb/train/sgd.pyc\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m     94\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0minp\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 96\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__update_grads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     97\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__update_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/Theano-0.7.0-py2.7.egg/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    604\u001b[0m                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_of_error\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    605\u001b[0m                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mthunks\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mposition_of_error\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 606\u001b[1;33m                         storage_map=self.fn.storage_map)\n\u001b[0m\u001b[0;32m    607\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    608\u001b[0m                     \u001b[1;31m# For the c linker We don't have access from\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/Theano-0.7.0-py2.7.egg/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    593\u001b[0m         \u001b[0mt0_fn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    594\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 595\u001b[1;33m             \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    596\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    597\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'position_of_error'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Output dimensions are not valid 1x-1\nApply node that caused the error: ConvOp{('imshp', (1, 50, None)),('kshp', (50, 3)),('nkern', 10),('bsize', 1),('dx', 1),('dy', 1),('out_mode', 'valid'),('unroll_batch', None),('unroll_kern', None),('unroll_patch', True),('imshp_logical', (1, 50, None)),('kshp_logical', (50, 3)),('kshp_logical_top_aligned', True)}(InplaceDimShuffle{x,x,1,0}.0, InplaceDimShuffle{0,x,1,2}.0)\nInputs types: [TensorType(float64, (True, True, False, False)), TensorType(float64, (False, True, False, False))]\nInputs shapes: [(1, 1, 50, 1), (10, 1, 50, 3)]\nInputs strides: [(400, 400, 8, 8), (1200, 1200, 24, 8)]\nInputs values: ['not shown', 'not shown']\n\nBacktrace when the node is created:\n  File \"build/bdist.linux-x86_64/egg/nnb/nn_model.py\", line 995, in apply\n    subsample=(1, stride)\n\nDebugprint of the apply node: \nConvOp{('imshp', (1, 50, None)),('kshp', (50, 3)),('nkern', 10),('bsize', 1),('dx', 1),('dy', 1),('out_mode', 'valid'),('unroll_batch', None),('unroll_kern', None),('unroll_patch', True),('imshp_logical', (1, 50, None)),('kshp_logical', (50, 3)),('kshp_logical_top_aligned', True)} [@A] <TensorType(float64, (True, False, True, False))> ''   \n |InplaceDimShuffle{x,x,1,0} [@B] <TensorType(float64, (True, True, False, False))> ''   \n | |input [@C] <TensorType(float64, matrix)>\n |InplaceDimShuffle{0,x,1,2} [@D] <TensorType(float64, (False, True, False, False))> ''   \n   |W [@E] <TensorType(float64, 3D)>\n\nStorage map footprint:\n - TensorConstant{-3}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - b, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - TensorConstant{1.0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - W, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - TensorConstant{(1, 1, 1) of 0.5}, Shape: (1, 1, 1), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - TensorConstant{-inf}, Shape: (1,), ElemSize: 4 Byte(s), TotalSize: 4.0 Byte(s)\n - <TensorType(float64, 3D)>, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - <TensorType(float64, 3D)>, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - b, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - b, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - Constant{-1}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - input, Shape: (461, 50), ElemSize: 8 Byte(s), TotalSize: 184400 Byte(s)\n - input, Shape: (1,), ElemSize: 4 Byte(s), TotalSize: 4 Byte(s)\n - b, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - Constant{0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - TensorConstant{0}, Shape: (1,), ElemSize: 1 Byte(s), TotalSize: 1.0 Byte(s)\n - TensorConstant{(1, 1, 1, 1) of 0.5}, Shape: (1, 1, 1, 1), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - MakeVector.0, Shape: (2,), ElemSize: 8 Byte(s), TotalSize: 16 Byte(s)\n - W, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - TensorConstant{(1,) of 2.0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - input, Shape: (1, 50), ElemSize: 8 Byte(s), TotalSize: 400 Byte(s)\n - W, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - input, Shape: (11, 50), ElemSize: 8 Byte(s), TotalSize: 4400 Byte(s)\n - TensorConstant{1}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - TensorConstant{(1,) of 0.5}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - max, Shape: (10, 10), ElemSize: 8 Byte(s), TotalSize: 800 Byte(s)\n - InplaceDimShuffle{x,1,x,3}.0, Shape: (1, 10, 1, 9), ElemSize: 8 Byte(s), TotalSize: 720 Byte(s)\n - Shape_i{2}.0, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - MakeVector.0, Shape: (2,), ElemSize: 8 Byte(s), TotalSize: 16 Byte(s)\n - Elemwise{Composite{(i0 + ((i1 + i2) // i0))}}[(0, 2)].0, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - Alloc.0, Shape: (10, 10), ElemSize: 8 Byte(s), TotalSize: 800 Byte(s)\n - TensorConstant{10}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - TensorConstant{0.0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8.0 Byte(s)\n - <TensorType(float64, vector)>, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - <TensorType(float64, vector)>, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - <TensorType(float64, matrix)>, Shape: (30, 1), ElemSize: 8 Byte(s), TotalSize: 240 Byte(s)\n - <TensorType(float64, vector)>, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - Reshape{3}.0, Shape: (10, 1, 10), ElemSize: 8 Byte(s), TotalSize: 800 Byte(s)\n - Elemwise{Composite{(i0 / Cast{float64}(i1))}}.0, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - <TensorType(float64, 3D)>, Shape: (10, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - Subtensor{int64}.0, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - W, Shape: (30, 1), ElemSize: 8 Byte(s), TotalSize: 240 Byte(s)\n - TensorConstant{(1, 1) of 0.0}, Shape: (1, 1), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - Alloc.0, Shape: (30,), ElemSize: 8 Byte(s), TotalSize: 240 Byte(s)\n - W.T, Shape: (1, 30), ElemSize: 8 Byte(s), TotalSize: 240 Byte(s)\n - TensorConstant{(1,) of 1.0}, Shape: (1,), ElemSize: 8 Byte(s), TotalSize: 8 Byte(s)\n - <TensorType(float64, vector)>, Shape: (10,), ElemSize: 8 Byte(s), TotalSize: 80 Byte(s)\n - InplaceDimShuffle{x,x,1,0}.0, Shape: (1, 1, 50, 11), ElemSize: 8 Byte(s), TotalSize: 4400 Byte(s)\n - InplaceDimShuffle{0,x,1,2}.0, Shape: (10, 1, 50, 3), ElemSize: 8 Byte(s), TotalSize: 12000 Byte(s)\n - InplaceDimShuffle{x,x,1,0}.0, Shape: (1, 1, 50, 1), ElemSize: 8 Byte(s), TotalSize: 400 Byte(s)\n\n"
     ]
    }
   ],
   "source": [
    "# CNN experiment\n",
    "\n",
    "L = []\n",
    "for i in range(3):\n",
    "    L.append(nnb.InputLayer(ndim=2) |\n",
    "             nnb.ConvolutionalLayer(insize=50, window=3, outsize=10, activation_func=nnb.activation.ReLU) |\n",
    "             nnb.MaxPoolingLayer(window=1)[0])\n",
    "\n",
    "CC = nnb.ConcatenationModel()\n",
    "C2 = nnb.PerceptronLayer(insize=3*10, outsize=1, activation_func=nnb.activation.tanh)\n",
    "model = (L[0] & L[1] & L[2]) | CC | C2\n",
    "model.compile()\n",
    "try_model(model, s0[:200], s1[:200], s2[:200], y[:200])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "theano.config.exception_verbosity = 'high'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import cPickle\n",
    "with open('model2.pkl', 'w') as f:\n",
    "    cPickle.dump(model2, f, protocol=cPickle.HIGHEST_PROTOCOL)\n",
    "with open('model_b200.pkl', 'w') as f:\n",
    "    cPickle.dump(model_b200, f, protocol=cPickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}