1dconvs_maxpool_256_onevsall.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:03:29.196238Z",
     "start_time": "2017-11-17T09:03:28.644004Z"
    },
    "_cell_guid": "679e0d3e-646d-4e96-9eb0-b362d8c6e51f",
    "_uuid": "0d05e5ce89af3e25d1c1fb244d021a1cfa1a058c"
   },
   "outputs": [],
   "source": [
    "from utils import *\n",
    "import array \n",
    "\n",
    "from pydub import AudioSegment\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Input, Conv2D, Flatten, GlobalMaxPooling1D, MaxPooling2D, Activation, BatchNormalization, GlobalAveragePooling2D, GlobalMaxPool2D, concatenate, Dense, Dropout\n",
    "from keras.optimizers import Adam\n",
    "from tensorflow.python.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Input, GRU, RepeatVector, BatchNormalization, TimeDistributed, Conv1D\n",
    "from keras.layers import GlobalAveragePooling1D, LSTM, MaxPooling1D, CuDNNLSTM, Bidirectional\n",
    "from keras import backend as K\n",
    "from keras.layers import  Conv2D, MaxPooling2D, UpSampling2D, Lambda, Reshape\n",
    "import keras\n",
    "from keras.layers import AveragePooling1D\n",
    "\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "import matplotlib.pyplot as plt\n",
    "import numba\n",
    "\n",
    "import dask.dataframe as dd\n",
    "from dask.multiprocessing import get\n",
    "from keras.layers import *\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:03:29.210749Z",
     "start_time": "2017-11-17T09:03:29.19832Z"
    },
    "_cell_guid": "8ab00801-08b9-44d3-a063-32e82dbf8f58",
    "_uuid": "53c19941676690454dd4b91109976b6c59cb7a40"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "POSSIBLE_LABELS = 'yes no up down left right on off stop go silence unknown'.split()\n",
    "id2name = {i: name for i, name in enumerate(POSSIBLE_LABELS)}\n",
    "name2id = {name: i for i, name in id2name.items()}\n",
    "len(id2name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13.9 s, sys: 18.7 s, total: 32.6 s\n",
      "Wall time: 32.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "train_df = pickle.load(open(\"cache/train_df_256_aug.pik\",\"rb\"))\n",
    "valid_df = pickle.load(open(\"cache/valid_df_256.pik\",\"rb\"))\n",
    "silent_df = pickle.load(open(\"cache/silent_df_256.pik\",\"rb\"))\n",
    "unknown_df = pickle.load(open(\"cache/unknown_df_256_aug.pik\",\"rb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.reset_index(inplace=True,drop=True)\n",
    "valid_df.reset_index(inplace=True,drop=True)\n",
    "unknown_df.reset_index(inplace=True,drop=True)\n",
    "silent_df.reset_index(inplace=True,drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:03:32.519795Z",
     "start_time": "2017-11-17T09:03:32.483881Z"
    },
    "_cell_guid": "144c6e60-8a83-437d-8b8a-ea065af90923",
    "_uuid": "22e0e6c718171167089fb6df36d3dc43a1029992"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def train_generator(train_batch_size,selected_class ):\n",
    "    \n",
    "\n",
    "\n",
    "    while True:\n",
    "        \n",
    "        this_train = train_df.groupby('label_id').apply(lambda x: x.sample(n = 2000))\n",
    "        extra_data_size = int(this_train.shape[0]* 0.1)\n",
    "\n",
    "        this_train = pd.concat([this_train,\n",
    "            silent_df.sample(extra_data_size) ,\n",
    "                                unknown_df.sample(extra_data_size*2)],axis=0 )\n",
    "        this_train[\"one_class_label\"] = (this_train.label_id == selected_class).astype(int)\n",
    "        \n",
    "\n",
    "        this_train.reset_index(drop=True,inplace=True)\n",
    "        shuffled_ids = random.sample(range(this_train.shape[0]), this_train.shape[0])\n",
    "        \n",
    "        for start in range(0, len(shuffled_ids), train_batch_size):\n",
    "            x_batch = []\n",
    "            y_batch = []\n",
    "            \n",
    "            end = min(start + train_batch_size, len(shuffled_ids))\n",
    "            i_train_batch = shuffled_ids[start:end]\n",
    "            for i in i_train_batch:\n",
    "                x_batch.append(this_train.loc[i,'raw'].T)\n",
    "                y_batch.append(this_train.one_class_label.values[i])\n",
    "                \n",
    "            x_batch = 1.- np.array(x_batch)/-80.\n",
    "            y_batch = np.array(y_batch) #to_categorical(y_batch, num_classes = len(POSSIBLE_LABELS))\n",
    "            \n",
    "            yield x_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 76 ms, sys: 0 ns, total: 76 ms\n",
      "Wall time: 75.7 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0])"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time t = next(train_generator(64,0))[1]\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:03:32.624289Z",
     "start_time": "2017-11-17T09:03:32.521828Z"
    },
    "_cell_guid": "59a13393-9bc3-4b27-abe2-9c78b3c32ead",
    "_uuid": "6f9a8fbf6e352b1c77b9c22dddf1c5d69382bd5b"
   },
   "outputs": [],
   "source": [
    "\n",
    "def valid_generator(val_batch_size,selected_class):\n",
    "    valid_df[\"one_class_label\"] = (valid_df.label_id == selected_class).astype(int)\n",
    "\n",
    "    while True:\n",
    "        ids = list(range(valid_df.shape[0]))\n",
    "        for start in range(0, len(ids), val_batch_size):\n",
    "            x_batch = []\n",
    "            y_batch = []\n",
    "            end = min(start + val_batch_size, len(ids))\n",
    "            i_val_batch = ids[start:end]\n",
    "            for i in i_val_batch:\n",
    "                x_batch.append(valid_df.loc[i,'raw'].T)\n",
    "                y_batch.append(valid_df.one_class_label.values[i])\n",
    "\n",
    "            x_batch = 1.- np.array(x_batch)/-80.\n",
    "            y_batch = np.array(y_batch) #to_categorical(y_batch, num_classes = len(POSSIBLE_LABELS))\n",
    "            yield x_batch, y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def x_generator(test_df, test_batch_size,augment=False):\n",
    "    while True:\n",
    "        ids = list(range(test_df.shape[0]))\n",
    "        \n",
    "        for start in range(0, len(ids), test_batch_size):\n",
    "            x_batch = []\n",
    "            end = min(start + test_batch_size, len(ids))\n",
    "            i_test_batch = ids[start:end]\n",
    "#             this_paths = test_paths[start:end]\n",
    "#             for x in this_paths:\n",
    "            for i in i_test_batch:\n",
    "                x_batch.append(test_df.loc[i,'raw'].T)\n",
    "            \n",
    "            x_batch = np.array(x_batch)\n",
    "            x_batch = 1.- np.array(x_batch)/-80.\n",
    "            yield x_batch\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0.\n",
    "\n",
    "def batch_relu(x):\n",
    "    x = BatchNormalization()(x)    \n",
    "    x = Activation('relu')(x)\n",
    "    \n",
    "    return x "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0.0\n",
    "\n",
    "def get_1dconvs_maxpool_freq_deep_fixed( x_in, kernel_size=10):\n",
    "    \n",
    "    x = BatchNormalization()(x_in)\n",
    "    \n",
    "    \n",
    "    x = Conv1D(64,kernel_size,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv1D(64,kernel_size,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    \n",
    "    \n",
    "    x = Dropout(p/2)(x)\n",
    "    x  = MaxPooling1D(2)(x)   \n",
    "\n",
    "\n",
    "\n",
    "    x = Conv1D(128,3,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv1D(128,3,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "\n",
    "    x = Dropout(p/2)(x)    \n",
    "    x  = MaxPooling1D(2,name=\"frozen\")(x) \n",
    "    \n",
    "\n",
    "    x = Conv1D(128,3,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv1D(128,3,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    \n",
    "    x = Dropout(p/2)(x)    \n",
    "    x  = MaxPooling1D(2)(x) \n",
    "    \n",
    "    \n",
    "    x = Conv1D(128,3,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    x = Conv1D(128,3,padding='same')(x)\n",
    "    x = batch_relu(x)\n",
    "    \n",
    "    x = Dropout(p/2)(x)     \n",
    "    x  = AveragePooling1D(2)(x) \n",
    "    \n",
    "    \n",
    "    \n",
    "    x_max = GlobalMaxPooling1D()(x)\n",
    "#     x_avg = GlobalAveragePooling1D()(x)\n",
    "\n",
    "#     x = concatenate([x_max,x_avg])\n",
    "    \n",
    "    return x_max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "timesteps, input_dim , latent_dim = 32,256, 128\n",
    "\n",
    "x_logml = Input(shape=(timesteps, input_dim)) #1 channel, 99 time, 161 freqs # S : np.ndarray [shape=(n_mels, t)]\n",
    "x_freq = Reshape((input_dim, timesteps))(x_logml)\n",
    "\n",
    "\n",
    "\n",
    "# 0.86\n",
    "# xf_2= get_1dconvs_maxpool_freq_deep(x_freq,2)\n",
    "xf_5 = get_1dconvs_maxpool_freq_deep_fixed(x_freq,10)\n",
    "\n",
    "x = Dense(128, activation = 'relu')(xf_5) #\n",
    "x = Dropout(0.3)(x)\n",
    "\n",
    "\n",
    "x = Dense(len(POSSIBLE_LABELS), activation = 'softmax', name='targets')(x)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# base_model = Model(inputs = x_logml, outputs = x)\n",
    "# \n",
    "\n",
    "# seen_frozen = False\n",
    "# for l in base_model.layers:\n",
    "#     if l.name == \"frozen\":\n",
    "#         seen_frozen = True\n",
    "     \n",
    "#     if not seen_frozen:\n",
    "#         l.trainable = False\n",
    "\n",
    "# base_model.compile(optimizer=Adam(lr=1e-3), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "# base_model.load_weights(\"weights/conv1d_2xunk_kernel10.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = Dense(1, activation = 'sigmoid', name='ova_target')(x) #(base_model.layers[-2].output)\n",
    "model = Model(inputs=[x_logml], outputs=[y])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=Adam(lr=1e-3), loss='binary_crossentropy', metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_class = 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_name = \"freq1d_2xunk_onevsall_class_{}\".format(selected_class)#max_freqconvs_2510_avgshortcuts\n",
    "callbacks = [EarlyStopping(monitor='val_loss',\n",
    "                           patience=5,\n",
    "                           verbose=1),\n",
    "             \n",
    "             ReduceLROnPlateau(monitor='val_loss',\n",
    "                               factor=0.1,\n",
    "                               patience=3,\n",
    "                               verbose=1,\n",
    "                               epsilon=0.01,\n",
    "                              min_lr=1e-5),\n",
    "             ModelCheckpoint(monitor='val_loss',\n",
    "                             filepath='weights/{}.hdf5'.format(exp_name),\n",
    "                             save_best_only=True,\n",
    "                             save_weights_only=True,\n",
    "                             mode='min')\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "cw=  1 / (11 + 2.) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9230769230769231"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1-cw"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Epoch 00005: reducing learning rate to 0.00010000000475.\n",
    "329/329 [==============================] - 10s 29ms/step - loss: 0.0059 - acc: 0.9844 - val_loss: 0.0108 - val_acc: 0.9966"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T09:42:31.48233Z",
     "start_time": "2017-11-17T09:03:33.355603Z"
    },
    "_cell_guid": "5f3d1b09-500f-410e-820a-8eaab24b6ebb",
    "_uuid": "528ec66a0a6caca952273ab916e609625839b19e",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "329/329 [==============================] - 22s 68ms/step - loss: 0.0784 - acc: 0.8030 - val_loss: 0.7615 - val_acc: 0.4847\n",
      "Epoch 2/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0702 - acc: 0.8159 - val_loss: 0.9628 - val_acc: 0.3094\n",
      "Epoch 3/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0628 - acc: 0.8484 - val_loss: 0.4774 - val_acc: 0.8369\n",
      "Epoch 4/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0557 - acc: 0.8825 - val_loss: 0.2837 - val_acc: 0.9512\n",
      "Epoch 5/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0515 - acc: 0.8886 - val_loss: 0.5038 - val_acc: 0.7960\n",
      "Epoch 6/100\n",
      "329/329 [==============================] - 17s 50ms/step - loss: 0.0506 - acc: 0.8906 - val_loss: 0.2893 - val_acc: 0.9369\n",
      "Epoch 7/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0421 - acc: 0.9192 - val_loss: 0.2403 - val_acc: 0.9459\n",
      "Epoch 8/100\n",
      "329/329 [==============================] - 17s 52ms/step - loss: 0.0415 - acc: 0.9164 - val_loss: 0.3774 - val_acc: 0.9130\n",
      "Epoch 9/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0370 - acc: 0.9352 - val_loss: 0.2219 - val_acc: 0.9545\n",
      "Epoch 10/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0329 - acc: 0.9404 - val_loss: 0.1167 - val_acc: 0.9916\n",
      "Epoch 11/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0301 - acc: 0.9485 - val_loss: 0.1561 - val_acc: 0.9761\n",
      "Epoch 12/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0272 - acc: 0.9553 - val_loss: 0.0987 - val_acc: 0.9894\n",
      "Epoch 13/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0266 - acc: 0.9555 - val_loss: 0.1280 - val_acc: 0.9827\n",
      "Epoch 14/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0240 - acc: 0.9596 - val_loss: 0.1232 - val_acc: 0.9735\n",
      "Epoch 15/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0230 - acc: 0.9539 - val_loss: 0.1464 - val_acc: 0.9609\n",
      "Epoch 16/100\n",
      "328/329 [============================>.] - ETA: 0s - loss: 0.0236 - acc: 0.9605\n",
      "Epoch 00016: reducing learning rate to 0.00010000000475.\n",
      "329/329 [==============================] - 17s 53ms/step - loss: 0.0235 - acc: 0.9606 - val_loss: 0.1431 - val_acc: 0.9712\n",
      "Epoch 17/100\n",
      "329/329 [==============================] - 17s 51ms/step - loss: 0.0184 - acc: 0.9735 - val_loss: 0.1185 - val_acc: 0.9787\n",
      "Epoch 00017: early stopping\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "\n",
    "history = model.fit_generator(generator=train_generator(batch_size,selected_class),\n",
    "                                      steps_per_epoch=train_df.shape[0]*(1./5)//batch_size,\n",
    "                                      epochs=100,\n",
    "                                      callbacks=callbacks,\n",
    "                                      validation_data=valid_generator(batch_size,selected_class),\n",
    "                                      validation_steps=int(np.ceil(valid_df.shape[0]/batch_size)), \n",
    "                                      class_weight={0:cw, 1:(1-cw)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "model.load_weights('./weights/{}.hdf5'.format(exp_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### valid evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('./weights/{}.hdf5'.format(exp_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_preds = model.predict_generator(valid_generator(64,selected_class),steps=int(np.ceil(valid_df.shape[0]/64.)))\n",
    "val_preds = val_preds > 0.5 # np.argmax(val_preds,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7055, 1)"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = (valid_df.label_id == selected_class).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.95      0.96       261\n",
      "         no       0.90      0.93      0.91       270\n",
      "         up       0.91      0.94      0.93       260\n",
      "       down       0.97      0.95      0.96       264\n",
      "       left       0.96      0.98      0.97       247\n",
      "      right       0.98      0.94      0.96       256\n",
      "         on       0.91      0.94      0.93       257\n",
      "        off       0.87      0.95      0.91       256\n",
      "       stop       0.97      0.95      0.96       246\n",
      "         go       0.92      0.89      0.91       260\n",
      "    silence       1.00      1.00      1.00       257\n",
      "    unknown       0.98      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_blend,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.95      0.97      0.96       261\n",
      "         no       0.88      0.93      0.91       270\n",
      "         up       0.90      0.93      0.91       260\n",
      "       down       0.95      0.97      0.96       264\n",
      "       left       0.92      0.98      0.95       247\n",
      "      right       0.94      0.97      0.96       256\n",
      "         on       0.90      0.95      0.93       257\n",
      "        off       0.87      0.97      0.92       256\n",
      "       stop       0.94      0.94      0.94       246\n",
      "         go       0.87      0.92      0.90       260\n",
      "    silence       0.99      1.00      1.00       257\n",
      "    unknown       0.99      0.96      0.97      4221\n",
      "\n",
      "avg / total       0.96      0.96      0.96      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "     others       0.99      0.99      0.99      6794\n",
      "        yes       0.87      0.85      0.86       261\n",
      "\n",
      "avg / total       0.99      0.99      0.99      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=[\"others\",id2name[selected_class]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "     others       1.00      1.00      1.00      6794\n",
      "        yes       0.95      0.97      0.96       261\n",
      "\n",
      "avg / total       1.00      1.00      1.00      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=[\"others\",id2name[selected_class]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.96      0.98       261\n",
      "         no       0.94      0.92      0.93       270\n",
      "         up       0.93      0.93      0.93       260\n",
      "       down       0.95      0.95      0.95       264\n",
      "       left       0.97      0.98      0.97       247\n",
      "      right       0.95      0.96      0.96       256\n",
      "         on       0.92      0.94      0.93       257\n",
      "        off       0.93      0.94      0.93       256\n",
      "       stop       0.96      0.95      0.95       246\n",
      "         go       0.93      0.90      0.92       260\n",
      "    silence       0.97      1.00      0.99       257\n",
      "    unknown       0.98      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.97      0.97      0.97       261\n",
      "         no       0.91      0.91      0.91       270\n",
      "         up       0.90      0.94      0.92       260\n",
      "       down       0.93      0.95      0.94       264\n",
      "       left       0.95      0.98      0.97       247\n",
      "      right       0.92      0.97      0.95       256\n",
      "         on       0.84      0.94      0.89       257\n",
      "        off       0.89      0.96      0.92       256\n",
      "       stop       0.94      0.96      0.95       246\n",
      "         go       0.86      0.92      0.89       260\n",
      "    silence       0.96      1.00      0.98       257\n",
      "    unknown       0.99      0.96      0.97      4221\n",
      "\n",
      "avg / total       0.96      0.96      0.96      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.97      0.97       261\n",
      "         no       0.87      0.93      0.90       270\n",
      "         up       0.86      0.93      0.89       260\n",
      "       down       0.92      0.94      0.93       264\n",
      "       left       0.91      0.99      0.95       247\n",
      "      right       0.93      0.96      0.94       256\n",
      "         on       0.88      0.94      0.91       257\n",
      "        off       0.88      0.94      0.91       256\n",
      "       stop       0.95      0.93      0.94       246\n",
      "         go       0.85      0.92      0.88       260\n",
      "    silence       0.97      1.00      0.98       257\n",
      "    unknown       0.99      0.96      0.97      4221\n",
      "\n",
      "avg / total       0.96      0.95      0.96      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 2xunk -timeconv-maxpool wlstm\n",
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.96      0.96      0.96       261\n",
      "         no       0.88      0.93      0.91       270\n",
      "         up       0.86      0.92      0.89       260\n",
      "       down       0.90      0.95      0.92       264\n",
      "       left       0.89      0.98      0.93       247\n",
      "      right       0.93      0.95      0.94       256\n",
      "         on       0.87      0.92      0.90       257\n",
      "        off       0.85      0.95      0.90       256\n",
      "       stop       0.95      0.94      0.94       246\n",
      "         go       0.83      0.90      0.86       260\n",
      "    silence       0.96      1.00      0.98       257\n",
      "    unknown       0.99      0.95      0.97      4221\n",
      "\n",
      "avg / total       0.95      0.95      0.95      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 2xunk -timeconv-maxpool\n",
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.96      0.95      0.96       261\n",
      "         no       0.88      0.91      0.90       270\n",
      "         up       0.80      0.95      0.87       260\n",
      "       down       0.96      0.89      0.93       264\n",
      "       left       0.90      0.96      0.93       247\n",
      "      right       0.90      0.93      0.92       256\n",
      "         on       0.93      0.88      0.90       257\n",
      "        off       0.90      0.90      0.90       256\n",
      "       stop       0.99      0.90      0.94       246\n",
      "         go       0.85      0.92      0.88       260\n",
      "    silence       0.97      1.00      0.98       257\n",
      "    unknown       0.98      0.97      0.97      4221\n",
      "\n",
      "avg / total       0.95      0.95      0.95      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 2x unknown\n",
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.97      0.97      0.97       261\n",
      "         no       0.91      0.92      0.91       270\n",
      "         up       0.83      0.95      0.89       260\n",
      "       down       0.95      0.96      0.95       264\n",
      "       left       0.93      0.98      0.96       247\n",
      "      right       0.90      0.96      0.93       256\n",
      "         on       0.88      0.95      0.91       257\n",
      "        off       0.90      0.95      0.92       256\n",
      "       stop       0.95      0.94      0.95       246\n",
      "         go       0.90      0.92      0.91       260\n",
      "    silence       0.98      1.00      0.99       257\n",
      "    unknown       0.99      0.96      0.97      4221\n",
      "\n",
      "avg / total       0.96      0.96      0.96      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 2x unknown\n",
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.97      0.98       261\n",
      "         no       0.92      0.94      0.93       270\n",
      "         up       0.93      0.95      0.94       260\n",
      "       down       0.98      0.97      0.97       264\n",
      "       left       0.96      0.99      0.97       247\n",
      "      right       0.96      0.96      0.96       256\n",
      "         on       0.93      0.96      0.94       257\n",
      "        off       0.90      0.97      0.93       256\n",
      "       stop       0.98      0.96      0.97       246\n",
      "         go       0.92      0.91      0.91       260\n",
      "    silence       0.99      1.00      1.00       257\n",
      "    unknown       0.99      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.96      0.97       261\n",
      "         no       0.90      0.92      0.91       270\n",
      "         up       0.91      0.94      0.92       260\n",
      "       down       0.95      0.96      0.96       264\n",
      "       left       0.95      0.96      0.96       247\n",
      "      right       0.96      0.97      0.96       256\n",
      "         on       0.91      0.95      0.93       257\n",
      "        off       0.92      0.94      0.93       256\n",
      "       stop       0.97      0.94      0.95       246\n",
      "         go       0.89      0.91      0.90       260\n",
      "    silence       0.98      1.00      0.99       257\n",
      "    unknown       0.98      0.97      0.98      4221\n",
      "\n",
      "avg / total       0.96      0.96      0.96      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.96      0.97       261\n",
      "         no       0.90      0.92      0.91       270\n",
      "         up       0.91      0.94      0.92       260\n",
      "       down       0.95      0.96      0.96       264\n",
      "       left       0.95      0.96      0.96       247\n",
      "      right       0.96      0.97      0.96       256\n",
      "         on       0.91      0.95      0.93       257\n",
      "        off       0.92      0.94      0.93       256\n",
      "       stop       0.97      0.94      0.95       246\n",
      "         go       0.89      0.91      0.90       260\n",
      "    silence       0.98      1.00      0.99       257\n",
      "    unknown       0.98      0.97      0.98      4221\n",
      "\n",
      "avg / total       0.96      0.96      0.96      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.97      0.97       261\n",
      "         no       0.92      0.94      0.93       270\n",
      "         up       0.89      0.93      0.91       260\n",
      "       down       0.96      0.97      0.96       264\n",
      "       left       0.95      0.99      0.97       247\n",
      "      right       0.96      0.97      0.97       256\n",
      "         on       0.91      0.95      0.93       257\n",
      "        off       0.92      0.95      0.93       256\n",
      "       stop       0.97      0.95      0.96       246\n",
      "         go       0.93      0.93      0.93       260\n",
      "    silence       0.99      1.00      0.99       257\n",
      "    unknown       0.99      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.96      0.97       261\n",
      "         no       0.91      0.94      0.92       270\n",
      "         up       0.93      0.92      0.92       260\n",
      "       down       0.93      0.97      0.95       264\n",
      "       left       0.95      0.98      0.96       247\n",
      "      right       0.95      0.95      0.95       256\n",
      "         on       0.93      0.93      0.93       257\n",
      "        off       0.87      0.95      0.91       256\n",
      "       stop       0.96      0.94      0.95       246\n",
      "         go       0.90      0.91      0.90       260\n",
      "    silence       0.99      1.00      0.99       257\n",
      "    unknown       0.98      0.97      0.98      4221\n",
      "\n",
      "avg / total       0.96      0.96      0.96      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.97      0.98       261\n",
      "         no       0.91      0.93      0.92       270\n",
      "         up       0.91      0.93      0.92       260\n",
      "       down       0.97      0.95      0.96       264\n",
      "       left       0.97      0.98      0.97       247\n",
      "      right       0.97      0.94      0.96       256\n",
      "         on       0.95      0.94      0.95       257\n",
      "        off       0.94      0.96      0.95       256\n",
      "       stop       0.97      0.91      0.94       246\n",
      "         go       0.88      0.87      0.87       260\n",
      "    silence       0.99      1.00      1.00       257\n",
      "    unknown       0.98      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.96      0.97       261\n",
      "         no       0.92      0.93      0.92       270\n",
      "         up       0.83      0.95      0.89       260\n",
      "       down       0.95      0.96      0.96       264\n",
      "       left       0.95      0.98      0.97       247\n",
      "      right       0.97      0.95      0.96       256\n",
      "         on       0.94      0.94      0.94       257\n",
      "        off       0.90      0.95      0.92       256\n",
      "       stop       0.96      0.94      0.95       246\n",
      "         go       0.92      0.92      0.92       260\n",
      "    silence       1.00      1.00      1.00       257\n",
      "    unknown       0.99      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.97      0.98       261\n",
      "         no       0.89      0.92      0.90       270\n",
      "         up       0.92      0.94      0.93       260\n",
      "       down       0.96      0.97      0.96       264\n",
      "       left       0.96      0.98      0.97       247\n",
      "      right       0.97      0.97      0.97       256\n",
      "         on       0.93      0.95      0.94       257\n",
      "        off       0.92      0.97      0.94       256\n",
      "       stop       0.97      0.94      0.96       246\n",
      "         go       0.91      0.91      0.91       260\n",
      "    silence       0.99      1.00      1.00       257\n",
      "    unknown       0.99      0.98      0.98      4221\n",
      "\n",
      "avg / total       0.97      0.97      0.97      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.96      0.97       261\n",
      "         no       0.88      0.91      0.89       270\n",
      "         up       0.86      0.95      0.90       260\n",
      "       down       0.92      0.95      0.93       264\n",
      "       left       0.94      0.98      0.96       247\n",
      "      right       0.94      0.95      0.95       256\n",
      "         on       0.83      0.94      0.88       257\n",
      "        off       0.87      0.94      0.90       256\n",
      "       stop       0.98      0.93      0.95       246\n",
      "         go       0.80      0.90      0.85       260\n",
      "    silence       0.95      1.00      0.98       257\n",
      "    unknown       0.98      0.95      0.97      4221\n",
      "\n",
      "avg / total       0.95      0.95      0.95      7055\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.96      0.97       261\n",
      "         no       0.92      0.92      0.92       270\n",
      "         up       0.93      0.93      0.93       260\n",
      "       down       0.99      0.95      0.97       264\n",
      "       left       0.98      0.97      0.97       247\n",
      "      right       0.99      0.95      0.97       256\n",
      "         on       0.97      0.92      0.95       257\n",
      "        off       0.94      0.92      0.93       256\n",
      "       stop       0.99      0.94      0.97       246\n",
      "         go       0.95      0.88      0.92       260\n",
      "    silence       0.97      1.00      0.98       257\n",
      "    unknown       0.76      0.98      0.86       257\n",
      "\n",
      "avg / total       0.95      0.94      0.94      3091\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.98      0.96      0.97       261\n",
      "         no       0.92      0.94      0.93       270\n",
      "         up       0.92      0.93      0.93       260\n",
      "       down       0.98      0.94      0.96       264\n",
      "       left       0.97      0.97      0.97       247\n",
      "      right       0.95      0.96      0.96       256\n",
      "         on       0.96      0.95      0.96       257\n",
      "        off       0.94      0.92      0.93       256\n",
      "       stop       1.00      0.94      0.97       246\n",
      "         go       0.96      0.89      0.92       260\n",
      "    silence       0.98      1.00      0.99       257\n",
      "    unknown       0.82      0.95      0.88       257\n",
      "\n",
      "avg / total       0.95      0.95      0.95      3091\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        yes       0.99      0.95      0.97       261\n",
      "         no       0.94      0.95      0.95       270\n",
      "         up       0.92      0.95      0.93       260\n",
      "       down       0.99      0.95      0.97       264\n",
      "       left       0.97      0.96      0.96       247\n",
      "      right       0.99      0.95      0.97       256\n",
      "         on       0.99      0.92      0.95       257\n",
      "        off       0.93      0.94      0.93       256\n",
      "       stop       0.98      0.95      0.97       246\n",
      "         go       0.94      0.90      0.92       260\n",
      "    silence       0.99      1.00      1.00       257\n",
      "    unknown       0.80      0.96      0.88       257\n",
      "\n",
      "avg / total       0.95      0.95      0.95      3091\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print classification_report(y_true,val_preds,target_names=POSSIBLE_LABELS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### submission\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T10:28:14.451612Z",
     "start_time": "2017-11-17T10:28:13.307142Z"
    },
    "_cell_guid": "72f27090-c0d1-4d0b-8027-34c915429a79",
    "_uuid": "1007977fccadecdae582ec5d8d52dd3c4c3010aa"
   },
   "outputs": [],
   "source": [
    "test_paths = glob(os.path.join('./data/', 'test/audio/*wav'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "158538"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pickle.load( open(\"cache/test_df_256.pik\",\"rb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158538, 5)"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T10:32:14.882322Z",
     "start_time": "2017-11-17T10:32:14.863617Z"
    },
    "_cell_guid": "c6d9b369-9979-4bcd-8540-4653e6544f84",
    "_uuid": "6a0bb3c22b7b5c43db0ec5673333ab3de8f08724"
   },
   "outputs": [],
   "source": [
    "def test_generator(test_batch_size,augment=False):\n",
    "    while True:\n",
    "        ids = list(range(test_df.shape[0]))\n",
    "        \n",
    "        for start in range(0, len(ids), test_batch_size):\n",
    "            x_batch = []\n",
    "            end = min(start + test_batch_size, len(ids))\n",
    "            i_test_batch = ids[start:end]\n",
    "#             this_paths = test_paths[start:end]\n",
    "#             for x in this_paths:\n",
    "            for i in i_test_batch:\n",
    "            #WATCHOUT > NO AUG\n",
    "#                 x_batch.append(process_wav_file(x).T) #,reshape=False,augment=augment,pval=0.5))\n",
    "                x_batch.append(test_df.loc[i,'raw'].T)\n",
    "\n",
    "            x_batch = np.array(x_batch)\n",
    "            x_batch = 1.- np.array(x_batch)/-80.\n",
    "            \n",
    "            yield x_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-11-17T10:32:45.947Z"
    },
    "_cell_guid": "1fb8aed4-de12-43c5-84bf-b803e3d640fa",
    "_uuid": "631a38cb0013e5772f6987854145ad76ecf6c430"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2478/2478 [==============================] - 67s 27ms/step\n"
     ]
    }
   ],
   "source": [
    "predictions = model.predict_generator(test_generator(batch_size), int(np.ceil(len(test_paths)/float(batch_size))), verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"cache/predictions_{}.npy\".format(exp_name),predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158538, 12)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### blending"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predictions_pyramid_noaug = np.load('cache/predictions_pyramid_noaug.npy')\n",
    "# predictions_model_with_ae_base_drp2_1 = np.load('cache/predictions_model_with_ae_base_drp2_1.npy')\n",
    "blend1 = np.load('cache/predictions_blend_dilated_conv1d_timek123_n_freqk48_pseudo_mixtimefreq1ds_plus_samewithlstm_plus_aebased_conv2d_finetuned.npy')\n",
    "# predictions_aebase_aug_drp3_finetune = np.load('cache/predictions_aebase_aug_drp3_finetune.npy')\n",
    "# predictions_conv_n_lstm = np.load(\"cache/predictions_dilated_conv_n_lstm.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "blend2 = np.mean([predictions,\n",
    "                       blend1], axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"cache/predictions_{}.npy\".format(\"frqmaxpool256_plus_longblend1\"),blend2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2477/2478 [============================>.] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "# num_aug = 2 \n",
    "# for i in range(num_aug):\n",
    "#     predictions +=  model.predict_generator(test_generator(64,augment=True), int(np.ceil(len(test_paths)/64.)), verbose=1)\n",
    "    \n",
    "\n",
    "# predictions = predictions/(num_aug + 1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### class assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T11:30:44.236246Z",
     "start_time": "2017-11-17T11:30:44.21858Z"
    },
    "_cell_guid": "b1cdab5c-9816-4690-87d8-de2c97cf0e7d",
    "_uuid": "24eb7e512eace4567494e0a8e356a826f4283c4d"
   },
   "outputs": [],
   "source": [
    "classes = np.argmax(predictions, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((158538,), 158538)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes.shape, len(test_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  11592.,    5042.,    5004.,    5542.,    5964.,    5972.,\n",
       "           6256.,    5536.,    5239.,  102391.]),\n",
       " array([  0. ,   1.1,   2.2,   3.3,   4.4,   5.5,   6.6,   7.7,   8.8,\n",
       "          9.9,  11. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEP9JREFUeJzt3W+oHXedx/H3ZxOrVbF/bAiahE1hgxILYg01bkGkkTZV\nMX2g0rJrgxTzwPoXwU19EvAPVBCrBS0UG5u6xVpqoUGj2VArsg9ae2vFmtbSS/8m25qr6R9X0Rr9\n7oPz6+7ZeJP8vOemk9y8X3A4M9/5zcx3SHI/d+bMnKSqkCSpxz8M3YAk6fhhaEiSuhkakqRuhoYk\nqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6rZ46Abm2xlnnFErV64cug1JOq7cc889v6mqJUcat+BC\nY+XKlUxNTQ3dhiQdV5I81jPOy1OSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZ\nGpKkbgvuiXBJGtLKzd8fZL+PXvmuF2U/RzzTSLI1yb4kvxyrnZ5kV5KH2vtprZ4kVyeZTvKLJGeP\nrbOxjX8oycax+puT3NfWuTpJDrcPSdJwei5PXQ+sP6i2Gbi9qlYBt7d5gAuBVe21CbgGRgEAbAHe\nApwDbBkLgWuAD42tt/4I+5AkDeSIoVFVPwH2H1TeAGxr09uAi8bqN9TIncCpSV4DXADsqqr9VfU0\nsAtY35a9qqrurKoCbjhoW7PtQ5I0kLl+EL60qp5s008BS9v0MuCJsXF7Wu1w9T2z1A+3D0nSQCa+\ne6qdIdQ89DLnfSTZlGQqydTMzMzRbEWSTmhzDY1ft0tLtPd9rb4XWDE2bnmrHa6+fJb64fbxN6rq\n2qpaU1Vrliw54v8hIkmao7mGxnbghTugNgK3jdUvbXdRrQWebZeYdgLnJzmtfQB+PrCzLXsuydp2\n19SlB21rtn1IkgZyxOc0knwbeDtwRpI9jO6CuhK4OcllwGPA+9vwHcA7gWngD8AHAapqf5LPAXe3\ncZ+tqhc+XP8wozu0TgZ+0F4cZh+SpIEcMTSq6pJDLFo3y9gCLj/EdrYCW2epTwFnzVL/7Wz7kCQN\nx68RkSR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS\n1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS\n1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUreJQiPJJ5PsTvLLJN9O8rIkZya5K8l0\nku8kOamNfWmbn27LV45t54pWfzDJBWP19a02nWTzJL1KkiY359BIsgz4GLCmqs4CFgEXA18Erqqq\nfwKeBi5rq1wGPN3qV7VxJFnd1nsDsB74epJFSRYBXwMuBFYDl7SxkqSBTHp5ajFwcpLFwMuBJ4Hz\ngFva8m3ARW16Q5unLV+XJK1+U1X9qaoeAaaBc9pruqoerqrngZvaWEnSQOYcGlW1F/gS8DijsHgW\nuAd4pqoOtGF7gGVtehnwRFv3QBv/6vH6Qescqi5JGsgkl6dOY/Sb/5nAa4FXMLq89KJLsinJVJKp\nmZmZIVqQpBPCJJen3gE8UlUzVfVn4FbgXODUdrkKYDmwt03vBVYAtOWnAL8drx+0zqHqf6Oqrq2q\nNVW1ZsmSJRMckiTpcCYJjceBtUle3j6bWAfcD9wBvLeN2Qjc1qa3t3na8h9VVbX6xe3uqjOBVcBP\ngbuBVe1urJMYfVi+fYJ+JUkTWnzkIbOrqruS3AL8DDgA3AtcC3wfuCnJ51vturbKdcC3kkwD+xmF\nAFW1O8nNjALnAHB5Vf0FIMlHgJ2M7szaWlW759qvJGlycw4NgKraAmw5qPwwozufDh77R+B9h9jO\nF4AvzFLfAeyYpEdJ0vzxiXBJUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlS\nN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlS\nN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0mCo0k\npya5JcmvkjyQ5K1JTk+yK8lD7f20NjZJrk4yneQXSc4e287GNv6hJBvH6m9Ocl9b5+okmaRfSdJk\nJj3T+Crww6p6PfBG4AFgM3B7Va0Cbm/zABcCq9prE3ANQJLTgS3AW4BzgC0vBE0b86Gx9dZP2K8k\naQJzDo0kpwBvA64DqKrnq+oZYAOwrQ3bBlzUpjcAN9TIncCpSV4DXADsqqr9VfU0sAtY35a9qqru\nrKoCbhjbliRpAJOcaZwJzADfTHJvkm8keQWwtKqebGOeApa26WXAE2Pr72m1w9X3zFL/G0k2JZlK\nMjUzMzPBIUmSDmeS0FgMnA1cU1VvAn7P/12KAqCdIdQE++hSVddW1ZqqWrNkyZKjvTtJOmFNEhp7\ngD1VdVebv4VRiPy6XVqive9ry/cCK8bWX95qh6svn6UuSRrInEOjqp4CnkjyulZaB9wPbAdeuANq\nI3Bbm94OXNruoloLPNsuY+0Ezk9yWvsA/HxgZ1v2XJK17a6pS8e2JUkawOIJ1/8ocGOSk4CHgQ8y\nCqKbk1wGPAa8v43dAbwTmAb+0MZSVfuTfA64u437bFXtb9MfBq4HTgZ+0F6SpIFMFBpV9XNgzSyL\n1s0ytoDLD7GdrcDWWepTwFmT9ChJmj8+ES5J6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRu\nhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRu\nhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRu\nE4dGkkVJ7k3yvTZ/ZpK7kkwn+U6Sk1r9pW1+ui1fObaNK1r9wSQXjNXXt9p0ks2T9ipJmsx8nGl8\nHHhgbP6LwFVV9U/A08BlrX4Z8HSrX9XGkWQ1cDHwBmA98PUWRIuArwEXAquBS9pYSdJAJgqNJMuB\ndwHfaPMBzgNuaUO2ARe16Q1tnrZ8XRu/Abipqv5UVY8A08A57TVdVQ9X1fPATW2sJGkgk55pfAX4\nNPDXNv9q4JmqOtDm9wDL2vQy4AmAtvzZNv5/6wetc6i6JGkgcw6NJO8G9lXVPfPYz1x72ZRkKsnU\nzMzM0O1I0oI1yZnGucB7kjzK6NLRecBXgVOTLG5jlgN72/ReYAVAW34K8Nvx+kHrHKr+N6rq2qpa\nU1VrlixZMsEhSZIOZ86hUVVXVNXyqlrJ6IPsH1XVvwB3AO9twzYCt7Xp7W2etvxHVVWtfnG7u+pM\nYBXwU+BuYFW7G+ukto/tc+1XkjS5xUce8nf7N+CmJJ8H7gWua/XrgG8lmQb2MwoBqmp3kpuB+4ED\nwOVV9ReAJB8BdgKLgK1Vtfso9CtJ6jQvoVFVPwZ+3KYfZnTn08Fj/gi87xDrfwH4wiz1HcCO+ehR\nkjQ5nwiXJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdD\nQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdD\nQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktRtzqGRZEWSO5Lcn2R3ko+3\n+ulJdiV5qL2f1upJcnWS6SS/SHL22LY2tvEPJdk4Vn9zkvvaOlcnySQHK0mazCRnGgeAT1XVamAt\ncHmS1cBm4PaqWgXc3uYBLgRWtdcm4BoYhQywBXgLcA6w5YWgaWM+NLbe+gn6lSRNaM6hUVVPVtXP\n2vTvgAeAZcAGYFsbtg24qE1vAG6okTuBU5O8BrgA2FVV+6vqaWAXsL4te1VV3VlVBdwwti1J0gDm\n5TONJCuBNwF3AUur6sm26ClgaZteBjwxttqeVjtcfc8s9dn2vynJVJKpmZmZiY5FknRoE4dGklcC\n3wU+UVXPjS9rZwg16T6OpKqurao1VbVmyZIlR3t3knTCmig0kryEUWDcWFW3tvKv26Ul2vu+Vt8L\nrBhbfXmrHa6+fJa6JGkgk9w9FeA64IGq+vLYou3AC3dAbQRuG6tf2u6iWgs82y5j7QTOT3Ja+wD8\nfGBnW/ZckrVtX5eObUuSNIDFE6x7LvAB4L4kP2+1zwBXAjcnuQx4DHh/W7YDeCcwDfwB+CBAVe1P\n8jng7jbus1W1v01/GLgeOBn4QXtJkgYy59Coqv8EDvXcxLpZxhdw+SG2tRXYOkt9Cjhrrj1KkuaX\nT4RLkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqdskX1i4\n4Kzc/P1B9vvole8aZL+S9PfyTEOS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdD\nQ5LUzdCQJHXza0SkE4BfkaP5YmjohDPUD1BpITA0NAh/cJ8Y/HNeeAyNY4D/sCQdL/wgXJLUzdCQ\nJHUzNCRJ3QwNSVI3Q0OS1O2YD40k65M8mGQ6yeah+5GkE9kxHRpJFgFfAy4EVgOXJFk9bFeSdOI6\npkMDOAeYrqqHq+p54CZgw8A9SdIJ61gPjWXAE2Pze1pNkjSABfFEeJJNwKY2+99JHpzjps4AfjM/\nXR1zFvKxwcI+Po/t+PWiHV++OPEm/rFn0LEeGnuBFWPzy1vt/6mqa4FrJ91ZkqmqWjPpdo5FC/nY\nYGEfn8d2/FqIx3esX566G1iV5MwkJwEXA9sH7kmSTljH9JlGVR1I8hFgJ7AI2FpVuwduS5JOWMd0\naABU1Q5gx4u0u4kvcR3DFvKxwcI+Po/t+LXgji9VNXQPkqTjxLH+mYYk6RhiaDQL9etKkqxIckeS\n+5PsTvLxoXuab0kWJbk3yfeG7mW+JTk1yS1JfpXkgSRvHbqn+ZLkk+3v5C+TfDvJy4buaRJJtibZ\nl+SXY7XTk+xK8lB7P23IHueDocGC/7qSA8Cnqmo1sBa4fAEd2ws+DjwwdBNHyVeBH1bV64E3skCO\nM8ky4GPAmqo6i9GNLhcP29XErgfWH1TbDNxeVauA29v8cc3QGFmwX1dSVU9W1c/a9O8Y/dBZME/V\nJ1kOvAv4xtC9zLckpwBvA64DqKrnq+qZYbuaV4uBk5MsBl4O/NfA/Uykqn4C7D+ovAHY1qa3ARe9\nqE0dBYbGyAnxdSVJVgJvAu4atpN59RXg08Bfh27kKDgTmAG+2S6/fSPJK4Zuaj5U1V7gS8DjwJPA\ns1X1H8N2dVQsraon2/RTwNIhm5kPhsYJIskrge8Cn6iq54buZz4keTewr6ruGbqXo2QxcDZwTVW9\nCfg9C+DyBkC7tr+BUTC+FnhFkn8dtqujq0a3qh73t6saGiNdX1dyvEryEkaBcWNV3Tp0P/PoXOA9\nSR5ldEnxvCT/PmxL82oPsKeqXjgzvIVRiCwE7wAeqaqZqvozcCvwzwP3dDT8OslrANr7voH7mZih\nMbJgv64kSRhdE3+gqr48dD/zqaquqKrlVbWS0Z/Zj6pqwfy2WlVPAU8keV0rrQPuH7Cl+fQ4sDbJ\ny9vf0XUskA/5D7Id2NimNwK3DdjLvDjmnwh/MSzwrys5F/gAcF+Sn7faZ9qT9jr2fRS4sf0y8zDw\nwYH7mRdVdVeSW4CfMbrD716O86enk3wbeDtwRpI9wBbgSuDmJJcBjwHvH67D+eET4ZKkbl6ekiR1\nMzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LU7X8AU5LCuztTGkUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fcc8286ed90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(classes,bins=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T11:31:11.212517Z",
     "start_time": "2017-11-17T11:31:10.786357Z"
    },
    "_cell_guid": "1da523cf-fdbf-4ab1-9300-0147155aa247",
    "_uuid": "f25d4e626202aa115bd0460f4de8d07f9727c83e"
   },
   "outputs": [],
   "source": [
    "### last batch will contain padding, so remove duplicates\n",
    "submission = dict()\n",
    "for i in range(len(test_paths)):\n",
    "    fname, label = os.path.basename(test_paths[i]), id2name[classes[i]]\n",
    "    submission[fname] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-17T11:32:05.154527Z",
     "start_time": "2017-11-17T11:32:04.983371Z"
    },
    "_cell_guid": "9a95d147-3f4b-4386-8597-5fa60be43542",
    "_uuid": "bdf63bce43a0525a02ac18ca3f90aeba06ce6e99"
   },
   "outputs": [],
   "source": [
    "with open('subm/{}.csv'.format(exp_name), 'w') as fout: #_blend_conv1dlstm_and_aebased_conv2d_finetuned\n",
    "    fout.write('fname,label\\n')\n",
    "    for fname, label in submission.items():\n",
    "        fout.write('{},{}\\n'.format(fname, label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "_cell_guid": "8bea6850-15c6-44e7-bdb4-9555ad196f85",
    "_uuid": "555315ef622793711ff5643928dac874c8cb0ed2"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='subm/timeconv5_freqconv10_wlstm128_pseudowval.csv' target='_blank'>subm/timeconv5_freqconv10_wlstm128_pseudowval.csv</a><br>"
      ],
      "text/plain": [
       "/home/ubuntu/keyword_spotting/subm/timeconv5_freqconv10_wlstm128_pseudowval.csv"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.lib.display import FileLink\n",
    "\n",
    "FileLink('subm/{}.csv'.format(exp_name))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}